From 4e94036ed7ba9f0075e7d9148ef258096048f7c9 Mon Sep 17 00:00:00 2001 From: pollopopo <61017252+pollopopo@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:57:23 +0200 Subject: [PATCH 1/3] feat(amf): add configurable max_au_size to cap encoded frame size Add a new optional AMF encoder parameter `amd_max_au_size` that caps the maximum encoded frame (Access Unit) size in bytes. When the AMF encoder (especially under CQP rate control) produces oversized frames, they can exceed the FEC shard limit (DATA_SHARDS_MAX=255 per block, max 4 blocks). When this happens, FEC error correction is skipped for those frames entirely (stream.cpp line 1373). On Wi-Fi networks, these large unprotected frames are highly susceptible to packet loss, causing visible dropped frames, stuttering, and stream instability. By setting `amd_max_au_size` (e.g. 800000 bytes), the hardware encoder itself enforces the frame size cap, ensuring every frame stays within FEC protection limits. This works with all rate control modes (CQP, CBR, VBR) and all three codecs (H.264, HEVC, AV1). Disabled by default (0 or empty) to preserve existing behavior. Tested on AMD RDNA4 (RX 9070 XT) streaming over 5GHz Wi-Fi with CQP mode. Setting max_au_size to 800000 eliminated FEC bypass warnings and significantly improved stream stability. --- docs/configuration.md | 30 +++++++++++++++++++ src/config.cpp | 2 ++ src/config.h | 1 + src/video.cpp | 3 ++ src_assets/common/assets/web/config.html | 1 + .../configs/tabs/encoders/AmdAmfEncoder.vue | 8 +++++ .../assets/web/public/assets/locale/en.json | 2 ++ 7 files changed, 47 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 97f08576cd1..588e24fdf15 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2788,6 +2788,36 @@ editing the `conf` file in a text editor. Use the examples as reference. +### amd_max_au_size + + + + + + + + + + + + + + +
Description + Maximum encoded frame size (Access Unit) in bytes. When set to a non-zero value, the + encoder will cap the maximum frame size. This prevents oversized frames from exceeding + FEC shard limits (DATA_SHARDS_MAX=255), which otherwise causes FEC to be skipped for + those frames. This is especially important for Wi-Fi streaming where packet loss on + large unprotected frames leads to dropped frames and stuttering. + @note{This option only applies when using amdvce [encoder](#encoder).} + @note{Works with all rate control modes (CQP, CBR, VBR). Most beneficial with CQP + where frame sizes are otherwise unbounded.} +
Default@code{} + + @endcode
Example@code{} + amd_max_au_size = 800000 + @endcode
+ ## VideoToolbox Encoder ### vt_coder diff --git a/src/config.cpp b/src/config.cpp index 47475a04b4d..17882839435 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -472,6 +472,7 @@ namespace config { (int) amd::quality_av1_e::balanced, // quality (av1) 0, // preanalysis 1, // vbaq + {}, // max_au_size (disabled by default) (int) amd::coder_e::_auto, // coder }, // amd @@ -1129,6 +1130,7 @@ namespace config { bool_f(vars, "amd_preanalysis", (bool &) video.amd.amd_preanalysis); bool_f(vars, "amd_vbaq", (bool &) video.amd.amd_vbaq); bool_f(vars, "amd_enforce_hrd", (bool &) video.amd.amd_enforce_hrd); + int_f(vars, "amd_max_au_size", video.amd.amd_max_au_size); int_f(vars, "vt_coder", video.vt.vt_coder, vt::coder_from_view); int_f(vars, "vt_software", video.vt.vt_allow_sw, vt::allow_software_from_view); diff --git a/src/config.h b/src/config.h index 6e4f001b707..01f55e42e8f 100644 --- a/src/config.h +++ b/src/config.h @@ -66,6 +66,7 @@ namespace config { std::optional amd_quality_av1; std::optional amd_preanalysis; std::optional amd_vbaq; + std::optional amd_max_au_size; int amd_coder; } amd; diff --git a/src/video.cpp b/src/video.cpp index 6b2b635277c..af8d57d7a6c 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -749,6 +749,7 @@ namespace video { {"rc"s, &config::video.amd.amd_rc_av1}, {"usage"s, &config::video.amd.amd_usage_av1}, {"enforce_hrd"s, &config::video.amd.amd_enforce_hrd}, + {"max_au_size"s, &config::video.amd.amd_max_au_size}, }, {}, // SDR-specific options {}, // HDR-specific options @@ -776,6 +777,7 @@ namespace video { {"usage"s, &config::video.amd.amd_usage_hevc}, {"vbaq"s, &config::video.amd.amd_vbaq}, {"enforce_hrd"s, &config::video.amd.amd_enforce_hrd}, + {"max_au_size"s, &config::video.amd.amd_max_au_size}, {"level"s, [](const config_t &cfg) { auto size = cfg.width * cfg.height; // For 4K and below, try to use level 5.1 or 5.2 if possible @@ -813,6 +815,7 @@ namespace video { {"usage"s, &config::video.amd.amd_usage_h264}, {"vbaq"s, &config::video.amd.amd_vbaq}, {"enforce_hrd"s, &config::video.amd.amd_enforce_hrd}, + {"max_au_size"s, &config::video.amd.amd_max_au_size}, }, {}, // SDR-specific options {}, // HDR-specific options diff --git a/src_assets/common/assets/web/config.html b/src_assets/common/assets/web/config.html index b391e3e7470..afbdb2f6ddf 100644 --- a/src_assets/common/assets/web/config.html +++ b/src_assets/common/assets/web/config.html @@ -314,6 +314,7 @@

{{ $t('config.configuration') }}

"amd_preanalysis": "disabled", "amd_vbaq": "enabled", "amd_coder": "auto", + "amd_max_au_size": "", }, }, { diff --git a/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue b/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue index 1da4999942c..86cdb394d9a 100644 --- a/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue +++ b/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue @@ -56,6 +56,14 @@ const config = ref(props.config) v-model="config.amd_enforce_hrd" default="false" > + + +
+ + +
{{ $t('config.amd_max_au_size_desc') }}
+
diff --git a/src_assets/common/assets/web/public/assets/locale/en.json b/src_assets/common/assets/web/public/assets/locale/en.json index a647f446b90..d1387c94cd4 100644 --- a/src_assets/common/assets/web/public/assets/locale/en.json +++ b/src_assets/common/assets/web/public/assets/locale/en.json @@ -109,6 +109,8 @@ "amd_coder_desc": "Allows you to select the entropy encoding to prioritize quality or encoding speed. H.264 only.", "amd_enforce_hrd": "AMF Hypothetical Reference Decoder (HRD) Enforcement", "amd_enforce_hrd_desc": "Increases the constraints on rate control to meet HRD model requirements. This greatly reduces bitrate overflows, but may cause encoding artifacts or reduced quality on certain cards.", + "amd_max_au_size": "AMF Max AU Size", + "amd_max_au_size_desc": "Maximum encoded frame size in bytes (0 or empty = disabled). Caps the largest frame the encoder can produce, preventing oversized frames from exceeding FEC limits. Recommended value: 800000 for Wi-Fi streaming with CQP mode.", "amd_preanalysis": "AMF Preanalysis", "amd_preanalysis_desc": "This enables rate-control preanalysis, which may increase quality at the expense of increased encoding latency.", "amd_quality": "AMF Quality", From e0f12e1000d39890b9c273fdbef3d3565984fc12 Mon Sep 17 00:00:00 2001 From: pollopopo <61017252+pollopopo@users.noreply.github.com> Date: Sun, 29 Mar 2026 20:33:54 +0200 Subject: [PATCH 2/3] fix: correct max_au_size units (bits not bytes), remove AV1 support Address review feedback: - max_au_size is in bits per FFmpeg AMF documentation, not bytes - AV1 encoder (av1_amf) does not support max_au_size, removed - Updated UI description, docs, and recommended value accordingly - FFmpeg default is -1 (disabled), aligned placeholder --- docs/configuration.md | 10 ++++++---- src/video.cpp | 1 - .../assets/web/configs/tabs/encoders/AmdAmfEncoder.vue | 2 +- .../common/assets/web/public/assets/locale/en.json | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 588e24fdf15..a519495e753 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2794,14 +2794,16 @@ editing the `conf` file in a text editor. Use the examples as reference. Description - Maximum encoded frame size (Access Unit) in bytes. When set to a non-zero value, the - encoder will cap the maximum frame size. This prevents oversized frames from exceeding + Maximum Access Unit (frame) size for rate control, in bits. When set, the encoder + will cap the maximum encoded frame size. This prevents oversized frames from exceeding FEC shard limits (DATA_SHARDS_MAX=255), which otherwise causes FEC to be skipped for those frames. This is especially important for Wi-Fi streaming where packet loss on large unprotected frames leads to dropped frames and stuttering. - @note{This option only applies when using amdvce [encoder](#encoder).} + @note{This option only applies to H.264 and HEVC when using amdvce [encoder](#encoder). + AV1 does not support this option.} @note{Works with all rate control modes (CQP, CBR, VBR). Most beneficial with CQP where frame sizes are otherwise unbounded.} + @note{Value is in bits. To cap at 800 KB, use 6400000 (800000 × 8).} @@ -2813,7 +2815,7 @@ editing the `conf` file in a text editor. Use the examples as reference. Example @code{} - amd_max_au_size = 800000 + amd_max_au_size = 6400000 @endcode diff --git a/src/video.cpp b/src/video.cpp index af8d57d7a6c..c5326c6d414 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -749,7 +749,6 @@ namespace video { {"rc"s, &config::video.amd.amd_rc_av1}, {"usage"s, &config::video.amd.amd_usage_av1}, {"enforce_hrd"s, &config::video.amd.amd_enforce_hrd}, - {"max_au_size"s, &config::video.amd.amd_max_au_size}, }, {}, // SDR-specific options {}, // HDR-specific options diff --git a/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue b/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue index 86cdb394d9a..738e2767258 100644 --- a/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue +++ b/src_assets/common/assets/web/configs/tabs/encoders/AmdAmfEncoder.vue @@ -61,7 +61,7 @@ const config = ref(props.config)
+ placeholder="-1" min="-1" v-model="config.amd_max_au_size" />
{{ $t('config.amd_max_au_size_desc') }}
diff --git a/src_assets/common/assets/web/public/assets/locale/en.json b/src_assets/common/assets/web/public/assets/locale/en.json index d1387c94cd4..0001785e87e 100644 --- a/src_assets/common/assets/web/public/assets/locale/en.json +++ b/src_assets/common/assets/web/public/assets/locale/en.json @@ -109,8 +109,8 @@ "amd_coder_desc": "Allows you to select the entropy encoding to prioritize quality or encoding speed. H.264 only.", "amd_enforce_hrd": "AMF Hypothetical Reference Decoder (HRD) Enforcement", "amd_enforce_hrd_desc": "Increases the constraints on rate control to meet HRD model requirements. This greatly reduces bitrate overflows, but may cause encoding artifacts or reduced quality on certain cards.", - "amd_max_au_size": "AMF Max AU Size", - "amd_max_au_size_desc": "Maximum encoded frame size in bytes (0 or empty = disabled). Caps the largest frame the encoder can produce, preventing oversized frames from exceeding FEC limits. Recommended value: 800000 for Wi-Fi streaming with CQP mode.", + "amd_max_au_size": "AMF Max AU Size (bits)", + "amd_max_au_size_desc": "Maximum Access Unit (frame) size in bits for rate control. Empty or -1 disables. Caps the largest frame the encoder can produce, preventing oversized frames from exceeding FEC limits on Wi-Fi. H.264 and HEVC only (AV1 not supported). Recommended: 6400000 (800 KB) for Wi-Fi streaming.", "amd_preanalysis": "AMF Preanalysis", "amd_preanalysis_desc": "This enables rate-control preanalysis, which may increase quality at the expense of increased encoding latency.", "amd_quality": "AMF Quality", From bc8906eed5cf484b685ff23842561669bdeae7c7 Mon Sep 17 00:00:00 2001 From: pollopopo <61017252+pollopopo@users.noreply.github.com> Date: Sun, 29 Mar 2026 20:39:29 +0200 Subject: [PATCH 3/3] fix: correct recommended max_au_size value and add sizing formula 800000 bits (~97 KB/frame, ~46 Mbps at 60fps) is the correct recommended value for Wi-Fi streaming, not 6400000. Added formula for users to calculate based on their bandwidth: (target_bitrate_mbps / fps / 1.2) * 1000000 where 1.2 accounts for ~20% FEC overhead. --- docs/configuration.md | 6 ++++-- src_assets/common/assets/web/public/assets/locale/en.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a519495e753..d41202aece4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2803,7 +2803,9 @@ editing the `conf` file in a text editor. Use the examples as reference. AV1 does not support this option.} @note{Works with all rate control modes (CQP, CBR, VBR). Most beneficial with CQP where frame sizes are otherwise unbounded.} - @note{Value is in bits. To cap at 800 KB, use 6400000 (800000 × 8).} + @note{Value is in bits. To calculate: (target_bitrate_mbps / fps / 1.2) × 1000000, + where 1.2 accounts for ~20% FEC overhead. Examples: 800000 for ~50 Mbps at 60fps, + 1600000 for ~100 Mbps at 60fps, 400000 for ~50 Mbps at 120fps.} @@ -2815,7 +2817,7 @@ editing the `conf` file in a text editor. Use the examples as reference. Example @code{} - amd_max_au_size = 6400000 + amd_max_au_size = 800000 @endcode diff --git a/src_assets/common/assets/web/public/assets/locale/en.json b/src_assets/common/assets/web/public/assets/locale/en.json index 0001785e87e..0b2448cd2e7 100644 --- a/src_assets/common/assets/web/public/assets/locale/en.json +++ b/src_assets/common/assets/web/public/assets/locale/en.json @@ -110,7 +110,7 @@ "amd_enforce_hrd": "AMF Hypothetical Reference Decoder (HRD) Enforcement", "amd_enforce_hrd_desc": "Increases the constraints on rate control to meet HRD model requirements. This greatly reduces bitrate overflows, but may cause encoding artifacts or reduced quality on certain cards.", "amd_max_au_size": "AMF Max AU Size (bits)", - "amd_max_au_size_desc": "Maximum Access Unit (frame) size in bits for rate control. Empty or -1 disables. Caps the largest frame the encoder can produce, preventing oversized frames from exceeding FEC limits on Wi-Fi. H.264 and HEVC only (AV1 not supported). Recommended: 6400000 (800 KB) for Wi-Fi streaming.", + "amd_max_au_size_desc": "Maximum Access Unit (frame) size in bits. Empty or -1 disables. Prevents oversized frames from exceeding FEC limits on Wi-Fi. H.264 and HEVC only. Formula: (target_bitrate_mbps / fps / 1.2) * 1000000. Example at 50 Mbps 60fps: (50/60/1.2)*1000000 = 694444. Recommended: 800000 for ~50 Mbps Wi-Fi.", "amd_preanalysis": "AMF Preanalysis", "amd_preanalysis_desc": "This enables rate-control preanalysis, which may increase quality at the expense of increased encoding latency.", "amd_quality": "AMF Quality",