prevent potential size_t overflow in UV plane size calculations

Author: rootvector2

src/codec_svt.c

@@ -278,9 +278,24 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,
 
 #if SVT_AV1_CHECK_VERSION(1, 8, 0)
         // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples.
-        const uint32_t uvWidth = (image->width + y_shift) >> y_shift;
-        const uint32_t uvRowBytes = uvWidth * bytesPerPixel;
-        const size_t uvSize = (size_t)uvRowBytes * uvHeight;
+        const size_t uvWidth = ((size_t)image->width + y_shift) >> y_shift;
+
+        // Use size_t to avoid 32-bit overflow
+        const size_t uvRowBytes = (size_t)uvWidth * (size_t)bytesPerPixel;
+
+        // Verify multiplication overflow
+        if (uvWidth != 0 &&
+            uvRowBytes / (size_t)uvWidth != (size_t)bytesPerPixel) {
+            goto cleanup;
+        }
+
+        const size_t uvSize = uvRowBytes * (size_t)uvHeight;
+
+        // Verify second multiplication overflow
+        if (uvHeight != 0 &&
+            uvSize / (size_t)uvHeight != uvRowBytes) {
+            goto cleanup;
+        }
         uvPlanes = avifAlloc(uvSize);
         if (uvPlanes == NULL) {
             goto cleanup;

src/reformat.c

@@ -1854,7 +1854,8 @@ void avifGetRGBAPixel(const avifRGBImage * src, uint32_t x, uint32_t y, const av
     assert(!src->isFloat || src->depth == 16);
     assert(src->format != AVIF_RGB_FORMAT_RGB_565 || src->depth == 8);
 
-    const uint8_t * const srcPixel = &src->pixels[(size_t)y * src->rowBytes + x * info->pixelBytes];
+    const size_t offset = (size_t)y * (size_t)src->rowBytes + (size_t)x * (size_t)info->pixelBytes;
+    const uint8_t * const srcPixel = &src->pixels[offset];
     if (info->channelBytes > 1) {
         uint16_t r = *((const uint16_t *)(&srcPixel[info->offsetBytesR]));
         uint16_t g = *((const uint16_t *)(&srcPixel[info->offsetBytesG]));