From 15c80fe552887400fbe2cd86408c07d9df0f6a29 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:07:14 +0000 Subject: [PATCH 01/13] fix(compositor): improve image overlay quality, caching, aspect ratio, and selectability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace nearest-neighbor prescaling with bilinear (image crate Triangle filter) for much better rendering of images containing text or fine detail - Cache decoded image overlays across UpdateParams calls — only re-decode when data_base64 or target rect dimensions change, reusing existing Arc otherwise - Lock aspect ratio for image layers during resize (same as video layers) - Show actual image thumbnail in compositor canvas UI for easier selection; switch border from dotted to solid, remove crosshatch pattern Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 34 ++++++++-- crates/nodes/src/video/compositor/overlay.rs | 28 ++++----- ui/src/components/CompositorCanvas.tsx | 66 ++++++++------------ ui/src/hooks/useCompositorLayers.ts | 13 ++-- 4 files changed, 74 insertions(+), 67 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 668f1cfe..d78a558e 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -666,14 +666,36 @@ impl CompositorNode { "Updating compositor config" ); - // Always re-decode image overlays (content may have changed - // even if the count is the same). + // Re-decode image overlays only when their content or + // target rect changed. When only video-layer positions + // are updated (the common case) the existing decoded + // bitmaps are reused via Arc, avoiding redundant base64 + // decode + bilinear prescale work. + let old_imgs = image_overlays.clone(); + let old_cfgs = &config.image_overlays; let mut new_image_overlays = Vec::with_capacity(new_config.image_overlays.len()); - for img_cfg in &new_config.image_overlays { - match decode_image_overlay(img_cfg) { - Ok(ov) => new_image_overlays.push(Arc::new(ov)), - Err(e) => tracing::warn!("Image overlay decode failed: {e}"), + for (i, img_cfg) in new_config.image_overlays.iter().enumerate() { + let reuse = old_cfgs.get(i).is_some_and(|old| { + old.data_base64 == img_cfg.data_base64 + && old.transform.rect.width == img_cfg.transform.rect.width + && old.transform.rect.height == img_cfg.transform.rect.height + }); + if reuse { + // Content and target dimensions unchanged — reuse + // the decoded bitmap, just update mutable transform + // fields (position, opacity, rotation, z_index). + let mut ov = (*old_imgs[i]).clone(); + ov.rect = img_cfg.transform.rect.clone(); + ov.opacity = img_cfg.transform.opacity; + ov.rotation_degrees = img_cfg.transform.rotation_degrees; + ov.z_index = img_cfg.transform.z_index; + new_image_overlays.push(Arc::new(ov)); + } else { + match decode_image_overlay(img_cfg) { + Ok(ov) => new_image_overlays.push(Arc::new(ov)), + Err(e) => tracing::warn!("Image overlay decode failed: {e}"), + } } } *image_overlays = Arc::from(new_image_overlays); diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index ffdfd401..4fb4ee8b 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -76,24 +76,18 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result Vec { - let sw = sw as usize; - let sh = sh as usize; - let dw = dw as usize; - let dh = dh as usize; - let mut out = vec![0u8; dw * dh * 4]; - for dy in 0..dh { - let sy = dy * sh / dh; - for dx in 0..dw { - let sx = dx * sw / dw; - let si = (sy * sw + sx) * 4; - let di = (dy * dw + dx) * 4; - out[di..di + 4].copy_from_slice(&src[si..si + 4]); - } - } - out + // SAFETY: caller guarantees src.len() == sw * sh * 4. + #[allow(clippy::expect_used)] + let src_img = image::RgbaImage::from_raw(sw, sh, src.to_vec()) + .expect("prescale_rgba: source dimensions do not match buffer length"); + let resized = image::imageops::resize(&src_img, dw, dh, image::imageops::FilterType::Triangle); + resized.into_raw() } // ── Bundled default font ──────────────────────────────────────────────────── diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index cb1431a2..9256e3bd 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -18,7 +18,7 @@ */ import styled from '@emotion/styled'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { LayerState, @@ -111,18 +111,6 @@ const InlineTextInput = styled.textarea` font-family: inherit; `; -/** Icon badge for image overlay layers */ -const ImageBadge = styled.div` - position: absolute; - inset: 0; - display: flex; - align-items: center; - justify-content: center; - pointer-events: none; - z-index: 1; - opacity: 0.5; -`; - const LayerDimensions = styled.div` position: absolute; bottom: 2px; @@ -201,26 +189,6 @@ const ResizeHandles: React.FC<{ )); ResizeHandles.displayName = 'ResizeHandles'; -// ── Image icon SVG ────────────────────────────────────────────────────────── - -const ImageIcon: React.FC<{ size?: number }> = ({ size = 24 }) => ( - - - - - -); - // ── Video input layer ─────────────────────────────────────────────────────── const VideoLayer: React.FC<{ @@ -437,6 +405,16 @@ const ImageOverlayLayer: React.FC<{ const borderColor = isSelected ? 'var(--sk-primary)' : `hsla(${hue}, 70%, 65%, 0.8)`; const bgColor = isSelected ? `hsla(${hue}, 60%, 50%, 0.25)` : `hsla(${hue}, 60%, 50%, 0.12)`; + // Build a data-URI for the image thumbnail. The overlay stores raw + // base64 — we guess the MIME type from the first bytes of the decoded + // header (PNG magic vs default JPEG). + const imgSrc = useMemo(() => { + if (!overlay.dataBase64) return undefined; + const isLikelyPng = overlay.dataBase64.startsWith('iVBOR'); + const mime = isLikelyPng ? 'image/png' : 'image/jpeg'; + return `data:${mime};base64,${overlay.dataBase64}`; + }, [overlay.dataBase64]); + return ( + {imgSrc && ( + {`Image + )} IMG #{index} - - - {isSelected && } ); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 722d4e6a..5661ca5c 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -632,11 +632,14 @@ export const useCompositorLayers = ( newY = orig.y + (orig.height - newH); } - // Constrain video layer resize to maintain aspect ratio. - // Text/image overlays (layerKind !== 'video') are not constrained here - // because text overlays no longer have resize handles and image overlays - // may intentionally use different dimensions. - if (state.layerKind === 'video' && orig.width > 0 && orig.height > 0) { + // Constrain video and image layer resize to maintain aspect ratio. + // Text overlays are not constrained here because they no longer have + // resize handles. + if ( + (state.layerKind === 'video' || state.layerKind === 'image') && + orig.width > 0 && + orig.height > 0 + ) { const ar = orig.width / orig.height; const isCorner = handle.length === 2; // 'ne', 'nw', 'se', 'sw' From ac36a3d1e295d61f7f8a74a138959d65b436ad61 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:08:23 +0000 Subject: [PATCH 02/13] fix(compositor): guard against index mismatch in image overlay cache Use old_imgs.get(i) instead of old_imgs[i] to avoid a panic when a previous decode_image_overlay call failed, leaving old_imgs shorter than old_cfgs. Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index d78a558e..22d46777 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -676,16 +676,21 @@ impl CompositorNode { let mut new_image_overlays = Vec::with_capacity(new_config.image_overlays.len()); for (i, img_cfg) in new_config.image_overlays.iter().enumerate() { - let reuse = old_cfgs.get(i).is_some_and(|old| { + // Check whether we can reuse an existing decoded + // bitmap: config content + target dimensions must + // match AND the old decoded overlay must exist at + // this index (it may be absent if a previous decode + // failed). + let reusable = old_cfgs.get(i).is_some_and(|old| { old.data_base64 == img_cfg.data_base64 && old.transform.rect.width == img_cfg.transform.rect.width && old.transform.rect.height == img_cfg.transform.rect.height }); - if reuse { + if let (true, Some(existing)) = (reusable, old_imgs.get(i)) { // Content and target dimensions unchanged — reuse // the decoded bitmap, just update mutable transform // fields (position, opacity, rotation, z_index). - let mut ov = (*old_imgs[i]).clone(); + let mut ov = (*existing).clone(); ov.rect = img_cfg.transform.rect.clone(); ov.opacity = img_cfg.transform.opacity; ov.rotation_degrees = img_cfg.transform.rotation_degrees; From 7915457dcf4eb824c9d0d7b32c800ef8b71afe7b Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:13:16 +0000 Subject: [PATCH 03/13] =?UTF-8?q?fix(compositor):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20proper=20index=20mapping=20for=20cache,=20broader?= =?UTF-8?q?=20MIME=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build a HashMap> by walking old configs and decoded overlays in tandem, so cache lookups use config index rather than assuming positional alignment (which breaks when a previous decode failed) - Add WebP and GIF magic-byte detection for image thumbnail data URIs Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 38 ++++++++++++++++++------ ui/src/components/CompositorCanvas.tsx | 10 ++++--- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 22d46777..841369e5 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -34,6 +34,7 @@ use config::CompositorConfig; use kernel::{CompositeResult, CompositeWorkItem, LayerSnapshot}; use overlay::{decode_image_overlay, rasterize_text_overlay, DecodedOverlay}; use schemars::schema_for; +use std::collections::HashMap; use std::sync::Arc; use streamkit_core::control::NodeControlMessage; use streamkit_core::pins::PinManagementMessage; @@ -671,26 +672,45 @@ impl CompositorNode { // are updated (the common case) the existing decoded // bitmaps are reused via Arc, avoiding redundant base64 // decode + bilinear prescale work. + // + // Build a lookup from the *successfully decoded* old + // overlays so we don't rely on positional index alignment + // (which breaks when a previous decode failed and left + // the decoded slice shorter than the config vec). let old_imgs = image_overlays.clone(); let old_cfgs = &config.image_overlays; + let mut old_by_cfg_idx: HashMap> = + HashMap::new(); + { + // Walk old configs and old decoded overlays in + // tandem: only configs whose decode succeeded have a + // corresponding entry in old_imgs. + let mut decoded_idx = 0usize; + for (cfg_idx, _cfg) in old_cfgs.iter().enumerate() { + if decoded_idx < old_imgs.len() { + old_by_cfg_idx.insert(cfg_idx, &old_imgs[decoded_idx]); + decoded_idx += 1; + } + } + } let mut new_image_overlays = Vec::with_capacity(new_config.image_overlays.len()); for (i, img_cfg) in new_config.image_overlays.iter().enumerate() { - // Check whether we can reuse an existing decoded - // bitmap: config content + target dimensions must - // match AND the old decoded overlay must exist at - // this index (it may be absent if a previous decode - // failed). - let reusable = old_cfgs.get(i).is_some_and(|old| { - old.data_base64 == img_cfg.data_base64 + let cached = old_cfgs.get(i).and_then(|old| { + if old.data_base64 == img_cfg.data_base64 && old.transform.rect.width == img_cfg.transform.rect.width && old.transform.rect.height == img_cfg.transform.rect.height + { + old_by_cfg_idx.get(&i) + } else { + None + } }); - if let (true, Some(existing)) = (reusable, old_imgs.get(i)) { + if let Some(existing) = cached { // Content and target dimensions unchanged — reuse // the decoded bitmap, just update mutable transform // fields (position, opacity, rotation, z_index). - let mut ov = (*existing).clone(); + let mut ov = (**existing).clone(); ov.rect = img_cfg.transform.rect.clone(); ov.opacity = img_cfg.transform.opacity; ov.rotation_degrees = img_cfg.transform.rotation_degrees; diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index 9256e3bd..9cd6a335 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -406,12 +406,14 @@ const ImageOverlayLayer: React.FC<{ const bgColor = isSelected ? `hsla(${hue}, 60%, 50%, 0.25)` : `hsla(${hue}, 60%, 50%, 0.12)`; // Build a data-URI for the image thumbnail. The overlay stores raw - // base64 — we guess the MIME type from the first bytes of the decoded - // header (PNG magic vs default JPEG). + // base64 — we detect the MIME type from the first bytes of the decoded + // header (magic-byte prefixes in base64 encoding). const imgSrc = useMemo(() => { if (!overlay.dataBase64) return undefined; - const isLikelyPng = overlay.dataBase64.startsWith('iVBOR'); - const mime = isLikelyPng ? 'image/png' : 'image/jpeg'; + let mime = 'image/jpeg'; // default fallback + if (overlay.dataBase64.startsWith('iVBOR')) mime = 'image/png'; + else if (overlay.dataBase64.startsWith('R0lGOD')) mime = 'image/gif'; + else if (overlay.dataBase64.startsWith('UklGR')) mime = 'image/webp'; return `data:${mime};base64,${overlay.dataBase64}`; }, [overlay.dataBase64]); From c3a5d6353b8b958b2a7d36c02543d4a671985a62 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:13:30 +0000 Subject: [PATCH 04/13] style(compositor): apply cargo fmt formatting Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 841369e5..7ed6f06c 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -679,8 +679,7 @@ impl CompositorNode { // the decoded slice shorter than the config vec). let old_imgs = image_overlays.clone(); let old_cfgs = &config.image_overlays; - let mut old_by_cfg_idx: HashMap> = - HashMap::new(); + let mut old_by_cfg_idx: HashMap> = HashMap::new(); { // Walk old configs and old decoded overlays in // tandem: only configs whose decode succeeded have a From 59faac5c2d2e3b08c5ba3cd53992cf2ab1dd1547 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:17:01 +0000 Subject: [PATCH 05/13] fix(compositor): fix HashMap type and double-deref in overlay cache Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 7ed6f06c..c08e913d 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -679,20 +679,21 @@ impl CompositorNode { // the decoded slice shorter than the config vec). let old_imgs = image_overlays.clone(); let old_cfgs = &config.image_overlays; - let mut old_by_cfg_idx: HashMap> = HashMap::new(); - { - // Walk old configs and old decoded overlays in - // tandem: only configs whose decode succeeded have a - // corresponding entry in old_imgs. - let mut decoded_idx = 0usize; - for (cfg_idx, _cfg) in old_cfgs.iter().enumerate() { - if decoded_idx < old_imgs.len() { - old_by_cfg_idx.insert(cfg_idx, &old_imgs[decoded_idx]); - decoded_idx += 1; - } + + // Map each *config* index to the corresponding decoded + // overlay. Failed decodes leave old_imgs shorter than + // old_cfgs, so we walk them in tandem rather than + // assuming positional alignment. + let mut old_by_cfg_idx: HashMap> = HashMap::new(); + let mut decoded_idx = 0usize; + for cfg_idx in 0..old_cfgs.len() { + if decoded_idx < old_imgs.len() { + old_by_cfg_idx.insert(cfg_idx, Arc::clone(&old_imgs[decoded_idx])); + decoded_idx += 1; } } - let mut new_image_overlays = + + let mut new_image_overlays: Vec> = Vec::with_capacity(new_config.image_overlays.len()); for (i, img_cfg) in new_config.image_overlays.iter().enumerate() { let cached = old_cfgs.get(i).and_then(|old| { From 4effd0d88af3aced5b651481aad74beb28768772 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:23:31 +0000 Subject: [PATCH 06/13] fix(compositor): content-keyed overlay cache with dimension-based matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace incorrect positional index mapping with a content-keyed cache that matches decoded overlays to configs by comparing prescaled bitmap dimensions against the config's target rect. This correctly handles the case where a mid-list decode failure makes the decoded slice shorter than the config vec — failed configs are skipped (not consumed) because their target dimensions won't match the next decoded overlay. Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 97 ++++++++++++++---------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index c08e913d..7f7ba3a0 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -673,55 +673,74 @@ impl CompositorNode { // bitmaps are reused via Arc, avoiding redundant base64 // decode + bilinear prescale work. // - // Build a lookup from the *successfully decoded* old - // overlays so we don't rely on positional index alignment - // (which breaks when a previous decode failed and left - // the decoded slice shorter than the config vec). + // We match cached overlays by content identity rather + // than positional index because failed decodes cause the + // decoded slice to be shorter than the config vec, + // making positional lookups incorrect. let old_imgs = image_overlays.clone(); let old_cfgs = &config.image_overlays; - // Map each *config* index to the corresponding decoded - // overlay. Failed decodes leave old_imgs shorter than - // old_cfgs, so we walk them in tandem rather than - // assuming positional alignment. - let mut old_by_cfg_idx: HashMap> = HashMap::new(); - let mut decoded_idx = 0usize; - for cfg_idx in 0..old_cfgs.len() { - if decoded_idx < old_imgs.len() { - old_by_cfg_idx.insert(cfg_idx, Arc::clone(&old_imgs[decoded_idx])); - decoded_idx += 1; + // Build a content-keyed cache from the successfully + // decoded old overlays. We match each decoded bitmap + // back to its originating config by walking old_cfgs + // and old_imgs together: for each old config we check + // whether the next decoded overlay's dimensions match + // the config's target rect (a successful decode + // prescales to exactly those dimensions). On mismatch + // we skip the config (it must have been a failed + // decode) without consuming a decoded entry. + // + // The cache key is (data_base64, width, height) and + // values are Vec to handle duplicate images at the + // same size. + type CacheKey<'a> = (&'a str, u32, u32); + let mut cache: HashMap, Vec>> = HashMap::new(); + let mut dec_iter = old_imgs.iter().peekable(); + for old_cfg in old_cfgs.iter() { + if let Some(decoded) = dec_iter.peek() { + let tw = old_cfg.transform.rect.width; + let th = old_cfg.transform.rect.height; + // A successfully decoded overlay is prescaled + // to the config's target dimensions. If the + // next decoded bitmap's size matches we know + // it belongs to this config; otherwise this + // config's decode must have failed. + if decoded.width == tw && decoded.height == th { + let key: CacheKey<'_> = (&old_cfg.data_base64, tw, th); + cache + .entry(key) + .or_default() + .push(Arc::clone(dec_iter.next().expect("peeked"))); + } } } let mut new_image_overlays: Vec> = Vec::with_capacity(new_config.image_overlays.len()); - for (i, img_cfg) in new_config.image_overlays.iter().enumerate() { - let cached = old_cfgs.get(i).and_then(|old| { - if old.data_base64 == img_cfg.data_base64 - && old.transform.rect.width == img_cfg.transform.rect.width - && old.transform.rect.height == img_cfg.transform.rect.height - { - old_by_cfg_idx.get(&i) - } else { - None - } - }); - if let Some(existing) = cached { - // Content and target dimensions unchanged — reuse - // the decoded bitmap, just update mutable transform - // fields (position, opacity, rotation, z_index). - let mut ov = (**existing).clone(); - ov.rect = img_cfg.transform.rect.clone(); - ov.opacity = img_cfg.transform.opacity; - ov.rotation_degrees = img_cfg.transform.rotation_degrees; - ov.z_index = img_cfg.transform.z_index; - new_image_overlays.push(Arc::new(ov)); - } else { - match decode_image_overlay(img_cfg) { - Ok(ov) => new_image_overlays.push(Arc::new(ov)), - Err(e) => tracing::warn!("Image overlay decode failed: {e}"), + for img_cfg in &new_config.image_overlays { + let key: CacheKey<'_> = ( + &img_cfg.data_base64, + img_cfg.transform.rect.width, + img_cfg.transform.rect.height, + ); + if let Some(entries) = cache.get_mut(&key) { + if let Some(existing) = entries.pop() { + // Content and target dimensions unchanged — + // reuse the decoded bitmap, just update + // mutable transform fields. + let mut ov = (*existing).clone(); + ov.rect = img_cfg.transform.rect.clone(); + ov.opacity = img_cfg.transform.opacity; + ov.rotation_degrees = img_cfg.transform.rotation_degrees; + ov.z_index = img_cfg.transform.z_index; + new_image_overlays.push(Arc::new(ov)); + continue; } } + match decode_image_overlay(img_cfg) { + Ok(ov) => new_image_overlays.push(Arc::new(ov)), + Err(e) => tracing::warn!("Image overlay decode failed: {e}"), + } } *image_overlays = Arc::from(new_image_overlays); From 6825ac2b1597e5c8a573a267fcee74b56e309ad8 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:37:12 +0000 Subject: [PATCH 07/13] fix(compositor): default image overlay z-index to 200 so it renders above video layers Co-Authored-By: Claudio Costa --- ui/src/hooks/useCompositorLayers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 5661ca5c..b9092e28 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -201,7 +201,7 @@ function parseImageOverlays(params: Record): ImageOverlayState[ height: o.rect?.height ?? 200, opacity: o.opacity ?? 1.0, rotationDegrees: o.rotation_degrees ?? 0, - zIndex: o.z_index ?? 0, + zIndex: o.z_index ?? (200 + i), visible: true, })); } @@ -1015,7 +1015,7 @@ export const useCompositorLayers = ( height: 200, opacity: 1.0, rotationDegrees: 0, - zIndex: 0, + zIndex: 200 + prev.length, visible: true, }, ]; From 3922a500a81cf896de4820e15fd2511b701c51da Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:42:38 +0000 Subject: [PATCH 08/13] style(compositor): add rationale comment for clippy::expect_used suppression Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/overlay.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index 4fb4ee8b..caac51dd 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -83,7 +83,7 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result Vec { // SAFETY: caller guarantees src.len() == sw * sh * 4. - #[allow(clippy::expect_used)] + #[allow(clippy::expect_used)] // from_raw only fails if buffer length != w*h*4; caller guarantees this let src_img = image::RgbaImage::from_raw(sw, sh, src.to_vec()) .expect("prescale_rgba: source dimensions do not match buffer length"); let resized = image::imageops::resize(&src_img, dw, dh, image::imageops::FilterType::Triangle); From 7094175b595967095caa42b241d90315bac8d3ac Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 17:42:57 +0000 Subject: [PATCH 09/13] style: apply formatting fixes (cargo fmt + prettier) Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/overlay.rs | 3 ++- ui/src/hooks/useCompositorLayers.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index caac51dd..b41fd2b5 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -83,7 +83,8 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result Vec { // SAFETY: caller guarantees src.len() == sw * sh * 4. - #[allow(clippy::expect_used)] // from_raw only fails if buffer length != w*h*4; caller guarantees this + #[allow(clippy::expect_used)] + // from_raw only fails if buffer length != w*h*4; caller guarantees this let src_img = image::RgbaImage::from_raw(sw, sh, src.to_vec()) .expect("prescale_rgba: source dimensions do not match buffer length"); let resized = image::imageops::resize(&src_img, dw, dh, image::imageops::FilterType::Triangle); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index b9092e28..8a478452 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -201,7 +201,7 @@ function parseImageOverlays(params: Record): ImageOverlayState[ height: o.rect?.height ?? 200, opacity: o.opacity ?? 1.0, rotationDegrees: o.rotation_degrees ?? 0, - zIndex: o.z_index ?? (200 + i), + zIndex: o.z_index ?? 200 + i, visible: true, })); } From e46fa893b143b27a084c069daeb725a36fa70cd4 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 2 Mar 2026 18:11:42 +0000 Subject: [PATCH 10/13] fix(compositor): address review findings #1-#4, #7 - Replace dimension-matching cache heuristic with index-based mapping using image_overlay_cfg_indices (finding #1) - Only update x/y position on cache hit, not full rect clone (finding #2) - Fix MIME sniffing comment wording to 'base64-encoded magic bytes', add BMP detection (finding #3) - Switch from data-URI to URL.createObjectURL with cleanup for image overlay thumbnails (finding #4) - Change SAFETY comment to Invariant in prescale_rgba (finding #7) Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 92 +++++++++++--------- crates/nodes/src/video/compositor/overlay.rs | 2 +- ui/src/components/CompositorCanvas.tsx | 27 +++++- 3 files changed, 74 insertions(+), 47 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 7f7ba3a0..d030fd10 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -253,8 +253,16 @@ impl ProcessorNode for CompositorNode { // Decode image overlays (once). Wrap in Arc so per-frame clones // into the work item are cheap reference-count bumps. + // + // `image_overlay_cfg_indices` records, for each successfully decoded + // overlay, the index of the originating `ImageOverlayConfig` in + // `config.image_overlays`. This allows the cache in + // `apply_update_params` to map decoded bitmaps back to their configs + // without relying on dimension-matching heuristics. let mut image_overlays_vec: Vec> = Vec::with_capacity(self.config.image_overlays.len()); + let mut image_overlay_cfg_indices: Vec = + Vec::with_capacity(self.config.image_overlays.len()); for (i, img_cfg) in self.config.image_overlays.iter().enumerate() { match decode_image_overlay(img_cfg) { Ok(overlay) => { @@ -269,6 +277,7 @@ impl ProcessorNode for CompositorNode { overlay.rect.height, ); image_overlays_vec.push(Arc::new(overlay)); + image_overlay_cfg_indices.push(i); }, Err(e) => { tracing::warn!("Failed to decode image overlay {}: {}", i, e); @@ -409,6 +418,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -490,6 +500,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -530,6 +541,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -652,6 +664,7 @@ impl CompositorNode { fn apply_update_params( config: &mut CompositorConfig, image_overlays: &mut Arc<[Arc]>, + image_overlay_cfg_indices: &mut Vec, text_overlays: &mut Arc<[Arc]>, params: serde_json::Value, stats_tracker: &mut NodeStatsTracker, @@ -673,76 +686,71 @@ impl CompositorNode { // bitmaps are reused via Arc, avoiding redundant base64 // decode + bilinear prescale work. // - // We match cached overlays by content identity rather - // than positional index because failed decodes cause the - // decoded slice to be shorter than the config vec, - // making positional lookups incorrect. + // The cache is keyed by (data_base64, width, height). + // `image_overlay_cfg_indices` provides an exact mapping + // from each decoded overlay back to its originating + // config index, eliminating any heuristic guessing + // about which decoded bitmap belongs to which config. let old_imgs = image_overlays.clone(); let old_cfgs = &config.image_overlays; - // Build a content-keyed cache from the successfully - // decoded old overlays. We match each decoded bitmap - // back to its originating config by walking old_cfgs - // and old_imgs together: for each old config we check - // whether the next decoded overlay's dimensions match - // the config's target rect (a successful decode - // prescales to exactly those dimensions). On mismatch - // we skip the config (it must have been a failed - // decode) without consuming a decoded entry. - // - // The cache key is (data_base64, width, height) and - // values are Vec to handle duplicate images at the - // same size. - type CacheKey<'a> = (&'a str, u32, u32); - let mut cache: HashMap, Vec>> = HashMap::new(); - let mut dec_iter = old_imgs.iter().peekable(); - for old_cfg in old_cfgs.iter() { - if let Some(decoded) = dec_iter.peek() { - let tw = old_cfg.transform.rect.width; - let th = old_cfg.transform.rect.height; - // A successfully decoded overlay is prescaled - // to the config's target dimensions. If the - // next decoded bitmap's size matches we know - // it belongs to this config; otherwise this - // config's decode must have failed. - if decoded.width == tw && decoded.height == th { - let key: CacheKey<'_> = (&old_cfg.data_base64, tw, th); - cache - .entry(key) - .or_default() - .push(Arc::clone(dec_iter.next().expect("peeked"))); + let mut cache: HashMap<(&str, u32, u32), Vec>> = + HashMap::new(); + + // Each decoded overlay has a recorded config index in + // `image_overlay_cfg_indices`. Use this to look up + // the originating config directly — no dimension + // matching needed. + for (dec_idx, decoded) in old_imgs.iter().enumerate() { + if let Some(&cfg_idx) = image_overlay_cfg_indices.get(dec_idx) { + if let Some(old_cfg) = old_cfgs.get(cfg_idx) { + let key = ( + old_cfg.data_base64.as_str(), + old_cfg.transform.rect.width, + old_cfg.transform.rect.height, + ); + cache.entry(key).or_default().push(Arc::clone(decoded)); } } } let mut new_image_overlays: Vec> = Vec::with_capacity(new_config.image_overlays.len()); - for img_cfg in &new_config.image_overlays { - let key: CacheKey<'_> = ( - &img_cfg.data_base64, + let mut new_cfg_indices: Vec = + Vec::with_capacity(new_config.image_overlays.len()); + for (new_idx, img_cfg) in new_config.image_overlays.iter().enumerate() { + let key = ( + img_cfg.data_base64.as_str(), img_cfg.transform.rect.width, img_cfg.transform.rect.height, ); if let Some(entries) = cache.get_mut(&key) { if let Some(existing) = entries.pop() { // Content and target dimensions unchanged — - // reuse the decoded bitmap, just update - // mutable transform fields. + // reuse the decoded bitmap. The cache key + // already guarantees width/height match, so + // only position and visual fields can differ. let mut ov = (*existing).clone(); - ov.rect = img_cfg.transform.rect.clone(); + ov.rect.x = img_cfg.transform.rect.x; + ov.rect.y = img_cfg.transform.rect.y; ov.opacity = img_cfg.transform.opacity; ov.rotation_degrees = img_cfg.transform.rotation_degrees; ov.z_index = img_cfg.transform.z_index; new_image_overlays.push(Arc::new(ov)); + new_cfg_indices.push(new_idx); continue; } } match decode_image_overlay(img_cfg) { - Ok(ov) => new_image_overlays.push(Arc::new(ov)), + Ok(ov) => { + new_image_overlays.push(Arc::new(ov)); + new_cfg_indices.push(new_idx); + }, Err(e) => tracing::warn!("Image overlay decode failed: {e}"), } } *image_overlays = Arc::from(new_image_overlays); + *image_overlay_cfg_indices = new_cfg_indices; // Re-rasterize text overlays. let new_text_overlays: Vec> = new_config diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index b41fd2b5..9f276f85 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -82,7 +82,7 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result Vec { - // SAFETY: caller guarantees src.len() == sw * sh * 4. + // Invariant: caller guarantees src.len() == sw * sh * 4. #[allow(clippy::expect_used)] // from_raw only fails if buffer length != w*h*4; caller guarantees this let src_img = image::RgbaImage::from_raw(sw, sh, src.to_vec()) diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index 9cd6a335..0ea65664 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -405,18 +405,37 @@ const ImageOverlayLayer: React.FC<{ const borderColor = isSelected ? 'var(--sk-primary)' : `hsla(${hue}, 70%, 65%, 0.8)`; const bgColor = isSelected ? `hsla(${hue}, 60%, 50%, 0.25)` : `hsla(${hue}, 60%, 50%, 0.12)`; - // Build a data-URI for the image thumbnail. The overlay stores raw - // base64 — we detect the MIME type from the first bytes of the decoded - // header (magic-byte prefixes in base64 encoding). + // Build an object URL for the image thumbnail. Using + // URL.createObjectURL avoids keeping a potentially large data-URI + // string in the DOM and is more memory-efficient for multi-MB images. + // + // MIME detection: we inspect the base64-encoded magic bytes at the + // start of the string to pick the correct MIME type. The fallback + // is JPEG, which covers the common `/9j/` prefix and variants. const imgSrc = useMemo(() => { if (!overlay.dataBase64) return undefined; let mime = 'image/jpeg'; // default fallback if (overlay.dataBase64.startsWith('iVBOR')) mime = 'image/png'; else if (overlay.dataBase64.startsWith('R0lGOD')) mime = 'image/gif'; else if (overlay.dataBase64.startsWith('UklGR')) mime = 'image/webp'; - return `data:${mime};base64,${overlay.dataBase64}`; + else if (overlay.dataBase64.startsWith('Qk')) mime = 'image/bmp'; + const binaryStr = atob(overlay.dataBase64); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + const blob = new Blob([bytes], { type: mime }); + return URL.createObjectURL(blob); }, [overlay.dataBase64]); + // Revoke the object URL when the component unmounts or the source + // changes to avoid memory leaks. + useEffect(() => { + return () => { + if (imgSrc) URL.revokeObjectURL(imgSrc); + }; + }, [imgSrc]); + return ( Date: Tue, 3 Mar 2026 11:28:39 +0000 Subject: [PATCH 11/13] fix(compositor): preserve image aspect ratio, add image layer controls, optimize base64 decode - Backend: prescale images with aspect-ratio preservation (scale-to-fit instead of stretch-to-fill) and centre within the target rect. - Backend: re-centre cached overlays on position update. - Frontend: detect natural image dimensions on add and set initial rect to match source aspect ratio. - Frontend: add opacity/rotation slider controls for selected image overlays (matching video and text layer controls). - Frontend: fix findAnyLayer to pass through rotationDegrees and zIndex for image overlays instead of hardcoding 0. - Frontend: replace O(n) atob + byte-by-byte loop with fetch(data-URI) for more efficient base64-to-blob conversion. - Frontend: remove BMP MIME detection (inconsistent browser support). - Frontend: add z-index band allocation comments (video 0-99, text 100-199, image 200+). Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/mod.rs | 15 ++-- crates/nodes/src/video/compositor/overlay.rs | 47 ++++++++++-- ui/src/components/CompositorCanvas.tsx | 49 +++++++----- ui/src/hooks/useCompositorLayers.ts | 27 +++++-- ui/src/nodes/CompositorNode.tsx | 79 ++++++++++++++++++-- 5 files changed, 172 insertions(+), 45 deletions(-) diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index d030fd10..b1bd4622 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -727,12 +727,17 @@ impl CompositorNode { if let Some(entries) = cache.get_mut(&key) { if let Some(existing) = entries.pop() { // Content and target dimensions unchanged — - // reuse the decoded bitmap. The cache key - // already guarantees width/height match, so - // only position and visual fields can differ. + // reuse the decoded bitmap. The overlay's + // rect may be smaller than the config rect + // due to aspect-ratio-preserving prescale, + // so re-centre within the new config rect. let mut ov = (*existing).clone(); - ov.rect.x = img_cfg.transform.rect.x; - ov.rect.y = img_cfg.transform.rect.y; + let cfg_w = img_cfg.transform.rect.width.cast_signed(); + let cfg_h = img_cfg.transform.rect.height.cast_signed(); + let ov_w = ov.rect.width.cast_signed(); + let ov_h = ov.rect.height.cast_signed(); + ov.rect.x = img_cfg.transform.rect.x + (cfg_w - ov_w) / 2; + ov.rect.y = img_cfg.transform.rect.y + (cfg_h - ov_h) / 2; ov.opacity = img_cfg.transform.opacity; ov.rotation_degrees = img_cfg.transform.rotation_degrees; ov.z_index = img_cfg.transform.z_index; diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index 9f276f85..b7d462cd 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -47,18 +47,49 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result 0 && target_h > 0 && (w != target_w || h != target_h) { + // Aspect-ratio-preserving fit: scale so the image fits inside + // the target box without distortion. + #[allow(clippy::cast_precision_loss)] + let scale = { + let sw = w as f32; + let sh = h as f32; + (target_w as f32 / sw).min(target_h as f32 / sh) + }; + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + let fit_w = ((w as f32 * scale).round() as u32).max(1); + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + let fit_h = ((h as f32 * scale).round() as u32).max(1); + let raw = rgba.into_raw(); - let scaled = prescale_rgba(&raw, w, h, target_w, target_h); + let scaled = prescale_rgba(&raw, w, h, fit_w, fit_h); + + // Adjust the rect to match the fitted dimensions so the blit + // stays on the identity-scale path and the image is centred + // within the originally requested area. + let mut rect = config.transform.rect.clone(); + rect.x += (target_w.cast_signed() - fit_w.cast_signed()) / 2; + rect.y += (target_h.cast_signed() - fit_h.cast_signed()) / 2; + rect.width = fit_w; + rect.height = fit_h; + Ok(DecodedOverlay { rgba_data: scaled, - width: target_w, - height: target_h, - rect: config.transform.rect.clone(), + width: fit_w, + height: fit_h, + rect, opacity: config.transform.opacity, rotation_degrees: config.transform.rotation_degrees, z_index: config.transform.z_index, diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index 0ea65664..d93703da 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -18,7 +18,7 @@ */ import styled from '@emotion/styled'; -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import type { LayerState, @@ -405,36 +405,45 @@ const ImageOverlayLayer: React.FC<{ const borderColor = isSelected ? 'var(--sk-primary)' : `hsla(${hue}, 70%, 65%, 0.8)`; const bgColor = isSelected ? `hsla(${hue}, 60%, 50%, 0.25)` : `hsla(${hue}, 60%, 50%, 0.12)`; - // Build an object URL for the image thumbnail. Using - // URL.createObjectURL avoids keeping a potentially large data-URI - // string in the DOM and is more memory-efficient for multi-MB images. + // Build a blob URL for the image thumbnail. Using fetch() with a + // data-URI lets the browser decode the base64 natively, which is + // more efficient than the manual atob() + byte-by-byte Uint8Array + // copy for large images. // // MIME detection: we inspect the base64-encoded magic bytes at the // start of the string to pick the correct MIME type. The fallback // is JPEG, which covers the common `/9j/` prefix and variants. - const imgSrc = useMemo(() => { - if (!overlay.dataBase64) return undefined; + const [imgSrc, setImgSrc] = useState(); + + useEffect(() => { + if (!overlay.dataBase64) { + setImgSrc(undefined); + return; + } let mime = 'image/jpeg'; // default fallback if (overlay.dataBase64.startsWith('iVBOR')) mime = 'image/png'; else if (overlay.dataBase64.startsWith('R0lGOD')) mime = 'image/gif'; else if (overlay.dataBase64.startsWith('UklGR')) mime = 'image/webp'; - else if (overlay.dataBase64.startsWith('Qk')) mime = 'image/bmp'; - const binaryStr = atob(overlay.dataBase64); - const bytes = new Uint8Array(binaryStr.length); - for (let i = 0; i < binaryStr.length; i++) { - bytes[i] = binaryStr.charCodeAt(i); - } - const blob = new Blob([bytes], { type: mime }); - return URL.createObjectURL(blob); - }, [overlay.dataBase64]); - // Revoke the object URL when the component unmounts or the source - // changes to avoid memory leaks. - useEffect(() => { + let cancelled = false; + let url: string | undefined; + + fetch(`data:${mime};base64,${overlay.dataBase64}`) + .then((r) => r.blob()) + .then((blob) => { + if (cancelled) return; + url = URL.createObjectURL(blob); + setImgSrc(url); + }) + .catch(() => { + // Ignore decode failures — no thumbnail shown. + }); + return () => { - if (imgSrc) URL.revokeObjectURL(imgSrc); + cancelled = true; + if (url) URL.revokeObjectURL(url); }; - }, [imgSrc]); + }, [overlay.dataBase64]); return ( void; updateTextOverlay: (id: string, updates: Partial>) => void; removeTextOverlay: (id: string) => void; - addImageOverlay: (dataBase64: string) => void; + addImageOverlay: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; updateImageOverlay: (id: string, updates: Partial>) => void; removeImageOverlay: (id: string) => void; } @@ -188,7 +188,9 @@ function parseTextOverlays(params: Record): TextOverlayState[] })); } -/** Parse image overlays from compositor params */ +/** Parse image overlays from compositor params. + * Z-index band: image overlays default to 200+i (video: 0–99, text: 100–199, + * image: 200+). */ function parseImageOverlays(params: Record): ImageOverlayState[] { const overlays = params.image_overlays as ImageOverlayConfig[] | undefined; if (!Array.isArray(overlays)) return []; @@ -476,8 +478,8 @@ export const useCompositorLayers = ( width: imgOverlay.width, height: imgOverlay.height, opacity: imgOverlay.opacity, - zIndex: 0, - rotationDegrees: 0, + zIndex: imgOverlay.zIndex, + rotationDegrees: imgOverlay.rotationDegrees, visible: imgOverlay.visible, }, kind: 'image', @@ -1002,8 +1004,18 @@ export const useCompositorLayers = ( // ── Image overlay CRUD ──────────────────────────────────────────────────── const addImageOverlay = useCallback( - (dataBase64: string) => { + (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => { setImageOverlays((prev) => { + // Compute initial rect that preserves source aspect ratio. + // Fit the image within a 200px box on its largest side. + const maxDim = 200; + let w = maxDim; + let h = maxDim; + if (naturalWidth && naturalHeight && naturalWidth > 0 && naturalHeight > 0) { + const scale = Math.min(maxDim / naturalWidth, maxDim / naturalHeight, 1); + w = Math.max(1, Math.round(naturalWidth * scale)); + h = Math.max(1, Math.round(naturalHeight * scale)); + } const next: ImageOverlayState[] = [ ...prev, { @@ -1011,10 +1023,11 @@ export const useCompositorLayers = ( dataBase64, x: 40, y: 40 + prev.length * 60, - width: 200, - height: 200, + width: w, + height: h, opacity: 1.0, rotationDegrees: 0, + // Z-index band: image overlays use 200+ (video: 0–99, text: 100–199) zIndex: 200 + prev.length, visible: true, }, diff --git a/ui/src/nodes/CompositorNode.tsx b/ui/src/nodes/CompositorNode.tsx index 76ff6ccc..9c899c70 100644 --- a/ui/src/nodes/CompositorNode.tsx +++ b/ui/src/nodes/CompositorNode.tsx @@ -435,7 +435,7 @@ const OverlayList: React.FC<{ onAddText: (text: string) => void; onUpdateText: (id: string, updates: Partial>) => void; onRemoveText: (id: string) => void; - onAddImage: (dataBase64: string) => void; + onAddImage: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; onRemoveImage: (id: string) => void; disabled: boolean; }> = React.memo( @@ -476,7 +476,13 @@ const OverlayList: React.FC<{ const result = reader.result as string; // Strip the data:image/...;base64, prefix const base64 = result.split(',')[1]; - if (base64) onAddImage(base64); + if (!base64) return; + // Detect natural dimensions so the initial rect preserves + // the source aspect ratio. + const img = new window.Image(); + img.onload = () => onAddImage(base64, img.naturalWidth, img.naturalHeight); + img.onerror = () => onAddImage(base64); + img.src = result; }; reader.readAsDataURL(file); e.target.value = ''; @@ -659,7 +665,8 @@ const UnifiedLayerList: React.FC<{ onAddText: (text: string) => void; onUpdateText: (id: string, updates: Partial>) => void; onRemoveText: (id: string) => void; - onAddImage: (dataBase64: string) => void; + onAddImage: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; + onUpdateImage: (id: string, updates: Partial>) => void; onRemoveImage: (id: string) => void; disabled: boolean; }> = React.memo( @@ -677,6 +684,7 @@ const UnifiedLayerList: React.FC<{ onUpdateText, onRemoveText, onAddImage, + onUpdateImage, onRemoveImage, disabled, }) => { @@ -727,8 +735,9 @@ const UnifiedLayerList: React.FC<{ const [menuOpen, setMenuOpen] = useState(false); const fileInputRef = useRef(null); - // Find selected text overlay for inline editing controls + // Find selected text/image overlay for inline editing controls const selectedTextOverlay = textOverlays.find((o) => o.id === selectedLayerId); + const selectedImageOverlay = imageOverlays.find((o) => o.id === selectedLayerId); const handleAddText = useCallback(() => { onAddText('Text'); @@ -750,7 +759,13 @@ const UnifiedLayerList: React.FC<{ reader.onload = () => { const result = reader.result as string; const base64 = result.split(',')[1]; - if (base64) onAddImage(base64); + if (!base64) return; + // Detect natural dimensions so the initial rect preserves + // the source aspect ratio. + const img = new window.Image(); + img.onload = () => onAddImage(base64, img.naturalWidth, img.naturalHeight); + img.onerror = () => onAddImage(base64); + img.src = result; }; reader.readAsDataURL(file); e.target.value = ''; @@ -1020,6 +1035,58 @@ const UnifiedLayerList: React.FC<{ )} + + {/* Controls for the selected image overlay */} + {selectedImageOverlay && ( + <> + + Image #{imageOverlays.indexOf(selectedImageOverlay)} + + ({Math.round(selectedImageOverlay.x)}, {Math.round(selectedImageOverlay.y)}) + + + + + Opacity + + onUpdateImage(selectedImageOverlay.id, { + opacity: Number.parseFloat(e.target.value), + }) + } + disabled={disabled} + className="nodrag nopan" + /> + {(selectedImageOverlay.opacity * 100).toFixed(0)}% + + + + Rotation + + onUpdateImage(selectedImageOverlay.id, { + rotationDegrees: Number.parseFloat(e.target.value), + }) + } + disabled={disabled} + className="nodrag nopan" + /> + {selectedImageOverlay.rotationDegrees.toFixed(0)}° + + + )} ); } @@ -1051,6 +1118,7 @@ const CompositorNode: React.FC = React.memo(({ id, data, se updateTextOverlay, removeTextOverlay, addImageOverlay, + updateImageOverlay, removeImageOverlay, } = useCompositorLayers({ nodeId: id, @@ -1136,6 +1204,7 @@ const CompositorNode: React.FC = React.memo(({ id, data, se onUpdateText={updateTextOverlay} onRemoveText={removeTextOverlay} onAddImage={addImageOverlay} + onUpdateImage={updateImageOverlay} onRemoveImage={removeImageOverlay} disabled={disabled} /> From 19e5cd0e522db502754dcc0d0062aa06167fa8e3 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Tue, 3 Mar 2026 13:40:54 +0000 Subject: [PATCH 12/13] fix(compositor): apply rotation transform to image overlay layer in canvas preview Co-Authored-By: Claudio Costa --- ui/src/components/CompositorCanvas.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index d93703da..ab011208 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -455,6 +455,8 @@ const ImageOverlayLayer: React.FC<{ width: overlay.width, height: overlay.height, opacity: overlay.visible ? overlay.opacity : 0.2, + transform: + overlay.rotationDegrees !== 0 ? `rotate(${overlay.rotationDegrees}deg)` : undefined, zIndex: overlay.zIndex ?? 200 + index, border: `2px solid ${borderColor}`, background: bgColor, From 291ca5a30fa8e4dc500aa3a01aa831b83bc87f61 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Tue, 3 Mar 2026 13:48:10 +0000 Subject: [PATCH 13/13] fix(compositor): include rotationDegrees and zIndex in overlay sync change detection Add rotationDegrees and zIndex to the image overlay change-detection comparisons in the params sync effect so that YAML or backend changes to these fields are reflected in the UI. Also add the missing zIndex check to the text overlay change detection for consistency. Co-Authored-By: Claudio Costa --- ui/src/hooks/useCompositorLayers.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index de710f5b..aaeeee93 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -408,6 +408,7 @@ export const useCompositorLayers = ( m.height !== currentText[i].height || m.opacity !== currentText[i].opacity || m.rotationDegrees !== currentText[i].rotationDegrees || + m.zIndex !== currentText[i].zIndex || m.text !== currentText[i].text || m.fontSize !== currentText[i].fontSize || m.visible !== currentText[i].visible @@ -438,6 +439,8 @@ export const useCompositorLayers = ( m.width !== currentImg[i].width || m.height !== currentImg[i].height || m.opacity !== currentImg[i].opacity || + m.rotationDegrees !== currentImg[i].rotationDegrees || + m.zIndex !== currentImg[i].zIndex || m.visible !== currentImg[i].visible ); return changed ? merged : currentImg;