diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 668f1cfe..b1bd4622 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -34,6 +34,7 @@ use config::CompositorConfig; use kernel::{CompositeResult, CompositeWorkItem, LayerSnapshot}; use overlay::{decode_image_overlay, rasterize_text_overlay, DecodedOverlay}; use schemars::schema_for; +use std::collections::HashMap; use std::sync::Arc; use streamkit_core::control::NodeControlMessage; use streamkit_core::pins::PinManagementMessage; @@ -252,8 +253,16 @@ impl ProcessorNode for CompositorNode { // Decode image overlays (once). Wrap in Arc so per-frame clones // into the work item are cheap reference-count bumps. + // + // `image_overlay_cfg_indices` records, for each successfully decoded + // overlay, the index of the originating `ImageOverlayConfig` in + // `config.image_overlays`. This allows the cache in + // `apply_update_params` to map decoded bitmaps back to their configs + // without relying on dimension-matching heuristics. let mut image_overlays_vec: Vec> = Vec::with_capacity(self.config.image_overlays.len()); + let mut image_overlay_cfg_indices: Vec = + Vec::with_capacity(self.config.image_overlays.len()); for (i, img_cfg) in self.config.image_overlays.iter().enumerate() { match decode_image_overlay(img_cfg) { Ok(overlay) => { @@ -268,6 +277,7 @@ impl ProcessorNode for CompositorNode { overlay.rect.height, ); image_overlays_vec.push(Arc::new(overlay)); + image_overlay_cfg_indices.push(i); }, Err(e) => { tracing::warn!("Failed to decode image overlay {}: {}", i, e); @@ -408,6 +418,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -489,6 +500,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -529,6 +541,7 @@ impl ProcessorNode for CompositorNode { Self::apply_update_params( &mut self.config, &mut image_overlays, + &mut image_overlay_cfg_indices, &mut text_overlays, params, &mut stats_tracker, @@ -651,6 +664,7 @@ impl CompositorNode { fn apply_update_params( config: &mut CompositorConfig, image_overlays: &mut Arc<[Arc]>, + image_overlay_cfg_indices: &mut Vec, text_overlays: &mut Arc<[Arc]>, params: serde_json::Value, stats_tracker: &mut NodeStatsTracker, @@ -666,17 +680,82 @@ impl CompositorNode { "Updating compositor config" ); - // Always re-decode image overlays (content may have changed - // even if the count is the same). - let mut new_image_overlays = + // Re-decode image overlays only when their content or + // target rect changed. When only video-layer positions + // are updated (the common case) the existing decoded + // bitmaps are reused via Arc, avoiding redundant base64 + // decode + bilinear prescale work. + // + // The cache is keyed by (data_base64, width, height). + // `image_overlay_cfg_indices` provides an exact mapping + // from each decoded overlay back to its originating + // config index, eliminating any heuristic guessing + // about which decoded bitmap belongs to which config. + let old_imgs = image_overlays.clone(); + let old_cfgs = &config.image_overlays; + + let mut cache: HashMap<(&str, u32, u32), Vec>> = + HashMap::new(); + + // Each decoded overlay has a recorded config index in + // `image_overlay_cfg_indices`. Use this to look up + // the originating config directly — no dimension + // matching needed. + for (dec_idx, decoded) in old_imgs.iter().enumerate() { + if let Some(&cfg_idx) = image_overlay_cfg_indices.get(dec_idx) { + if let Some(old_cfg) = old_cfgs.get(cfg_idx) { + let key = ( + old_cfg.data_base64.as_str(), + old_cfg.transform.rect.width, + old_cfg.transform.rect.height, + ); + cache.entry(key).or_default().push(Arc::clone(decoded)); + } + } + } + + let mut new_image_overlays: Vec> = + Vec::with_capacity(new_config.image_overlays.len()); + let mut new_cfg_indices: Vec = Vec::with_capacity(new_config.image_overlays.len()); - for img_cfg in &new_config.image_overlays { + for (new_idx, img_cfg) in new_config.image_overlays.iter().enumerate() { + let key = ( + img_cfg.data_base64.as_str(), + img_cfg.transform.rect.width, + img_cfg.transform.rect.height, + ); + if let Some(entries) = cache.get_mut(&key) { + if let Some(existing) = entries.pop() { + // Content and target dimensions unchanged — + // reuse the decoded bitmap. The overlay's + // rect may be smaller than the config rect + // due to aspect-ratio-preserving prescale, + // so re-centre within the new config rect. + let mut ov = (*existing).clone(); + let cfg_w = img_cfg.transform.rect.width.cast_signed(); + let cfg_h = img_cfg.transform.rect.height.cast_signed(); + let ov_w = ov.rect.width.cast_signed(); + let ov_h = ov.rect.height.cast_signed(); + ov.rect.x = img_cfg.transform.rect.x + (cfg_w - ov_w) / 2; + ov.rect.y = img_cfg.transform.rect.y + (cfg_h - ov_h) / 2; + ov.opacity = img_cfg.transform.opacity; + ov.rotation_degrees = img_cfg.transform.rotation_degrees; + ov.z_index = img_cfg.transform.z_index; + new_image_overlays.push(Arc::new(ov)); + new_cfg_indices.push(new_idx); + continue; + } + } match decode_image_overlay(img_cfg) { - Ok(ov) => new_image_overlays.push(Arc::new(ov)), + Ok(ov) => { + new_image_overlays.push(Arc::new(ov)); + new_cfg_indices.push(new_idx); + }, Err(e) => tracing::warn!("Image overlay decode failed: {e}"), } } *image_overlays = Arc::from(new_image_overlays); + *image_overlay_cfg_indices = new_cfg_indices; // Re-rasterize text overlays. let new_text_overlays: Vec> = new_config diff --git a/crates/nodes/src/video/compositor/overlay.rs b/crates/nodes/src/video/compositor/overlay.rs index ffdfd401..b7d462cd 100644 --- a/crates/nodes/src/video/compositor/overlay.rs +++ b/crates/nodes/src/video/compositor/overlay.rs @@ -47,18 +47,49 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result 0 && target_h > 0 && (w != target_w || h != target_h) { + // Aspect-ratio-preserving fit: scale so the image fits inside + // the target box without distortion. + #[allow(clippy::cast_precision_loss)] + let scale = { + let sw = w as f32; + let sh = h as f32; + (target_w as f32 / sw).min(target_h as f32 / sh) + }; + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + let fit_w = ((w as f32 * scale).round() as u32).max(1); + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + let fit_h = ((h as f32 * scale).round() as u32).max(1); + let raw = rgba.into_raw(); - let scaled = prescale_rgba(&raw, w, h, target_w, target_h); + let scaled = prescale_rgba(&raw, w, h, fit_w, fit_h); + + // Adjust the rect to match the fitted dimensions so the blit + // stays on the identity-scale path and the image is centred + // within the originally requested area. + let mut rect = config.transform.rect.clone(); + rect.x += (target_w.cast_signed() - fit_w.cast_signed()) / 2; + rect.y += (target_h.cast_signed() - fit_h.cast_signed()) / 2; + rect.width = fit_w; + rect.height = fit_h; + Ok(DecodedOverlay { rgba_data: scaled, - width: target_w, - height: target_h, - rect: config.transform.rect.clone(), + width: fit_w, + height: fit_h, + rect, opacity: config.transform.opacity, rotation_degrees: config.transform.rotation_degrees, z_index: config.transform.z_index, @@ -76,24 +107,19 @@ pub fn decode_image_overlay(config: &ImageOverlayConfig) -> Result Vec { - let sw = sw as usize; - let sh = sh as usize; - let dw = dw as usize; - let dh = dh as usize; - let mut out = vec![0u8; dw * dh * 4]; - for dy in 0..dh { - let sy = dy * sh / dh; - for dx in 0..dw { - let sx = dx * sw / dw; - let si = (sy * sw + sx) * 4; - let di = (dy * dw + dx) * 4; - out[di..di + 4].copy_from_slice(&src[si..si + 4]); - } - } - out + // Invariant: caller guarantees src.len() == sw * sh * 4. + #[allow(clippy::expect_used)] + // from_raw only fails if buffer length != w*h*4; caller guarantees this + let src_img = image::RgbaImage::from_raw(sw, sh, src.to_vec()) + .expect("prescale_rgba: source dimensions do not match buffer length"); + let resized = image::imageops::resize(&src_img, dw, dh, image::imageops::FilterType::Triangle); + resized.into_raw() } // ── Bundled default font ──────────────────────────────────────────────────── diff --git a/ui/src/components/CompositorCanvas.tsx b/ui/src/components/CompositorCanvas.tsx index cb1431a2..ab011208 100644 --- a/ui/src/components/CompositorCanvas.tsx +++ b/ui/src/components/CompositorCanvas.tsx @@ -111,18 +111,6 @@ const InlineTextInput = styled.textarea` font-family: inherit; `; -/** Icon badge for image overlay layers */ -const ImageBadge = styled.div` - position: absolute; - inset: 0; - display: flex; - align-items: center; - justify-content: center; - pointer-events: none; - z-index: 1; - opacity: 0.5; -`; - const LayerDimensions = styled.div` position: absolute; bottom: 2px; @@ -201,26 +189,6 @@ const ResizeHandles: React.FC<{ )); ResizeHandles.displayName = 'ResizeHandles'; -// ── Image icon SVG ────────────────────────────────────────────────────────── - -const ImageIcon: React.FC<{ size?: number }> = ({ size = 24 }) => ( - - - - - -); - // ── Video input layer ─────────────────────────────────────────────────────── const VideoLayer: React.FC<{ @@ -437,6 +405,46 @@ const ImageOverlayLayer: React.FC<{ const borderColor = isSelected ? 'var(--sk-primary)' : `hsla(${hue}, 70%, 65%, 0.8)`; const bgColor = isSelected ? `hsla(${hue}, 60%, 50%, 0.25)` : `hsla(${hue}, 60%, 50%, 0.12)`; + // Build a blob URL for the image thumbnail. Using fetch() with a + // data-URI lets the browser decode the base64 natively, which is + // more efficient than the manual atob() + byte-by-byte Uint8Array + // copy for large images. + // + // MIME detection: we inspect the base64-encoded magic bytes at the + // start of the string to pick the correct MIME type. The fallback + // is JPEG, which covers the common `/9j/` prefix and variants. + const [imgSrc, setImgSrc] = useState(); + + useEffect(() => { + if (!overlay.dataBase64) { + setImgSrc(undefined); + return; + } + let mime = 'image/jpeg'; // default fallback + if (overlay.dataBase64.startsWith('iVBOR')) mime = 'image/png'; + else if (overlay.dataBase64.startsWith('R0lGOD')) mime = 'image/gif'; + else if (overlay.dataBase64.startsWith('UklGR')) mime = 'image/webp'; + + let cancelled = false; + let url: string | undefined; + + fetch(`data:${mime};base64,${overlay.dataBase64}`) + .then((r) => r.blob()) + .then((blob) => { + if (cancelled) return; + url = URL.createObjectURL(blob); + setImgSrc(url); + }) + .catch(() => { + // Ignore decode failures — no thumbnail shown. + }); + + return () => { + cancelled = true; + if (url) URL.revokeObjectURL(url); + }; + }, [overlay.dataBase64]); + return ( + {imgSrc && ( + {`Image + )} IMG #{index} - - - {isSelected && } ); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 722d4e6a..aaeeee93 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -107,7 +107,7 @@ export interface UseCompositorLayersResult { addTextOverlay: (text: string) => void; updateTextOverlay: (id: string, updates: Partial>) => void; removeTextOverlay: (id: string) => void; - addImageOverlay: (dataBase64: string) => void; + addImageOverlay: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; updateImageOverlay: (id: string, updates: Partial>) => void; removeImageOverlay: (id: string) => void; } @@ -188,7 +188,9 @@ function parseTextOverlays(params: Record): TextOverlayState[] })); } -/** Parse image overlays from compositor params */ +/** Parse image overlays from compositor params. + * Z-index band: image overlays default to 200+i (video: 0–99, text: 100–199, + * image: 200+). */ function parseImageOverlays(params: Record): ImageOverlayState[] { const overlays = params.image_overlays as ImageOverlayConfig[] | undefined; if (!Array.isArray(overlays)) return []; @@ -201,7 +203,7 @@ function parseImageOverlays(params: Record): ImageOverlayState[ height: o.rect?.height ?? 200, opacity: o.opacity ?? 1.0, rotationDegrees: o.rotation_degrees ?? 0, - zIndex: o.z_index ?? 0, + zIndex: o.z_index ?? 200 + i, visible: true, })); } @@ -406,6 +408,7 @@ export const useCompositorLayers = ( m.height !== currentText[i].height || m.opacity !== currentText[i].opacity || m.rotationDegrees !== currentText[i].rotationDegrees || + m.zIndex !== currentText[i].zIndex || m.text !== currentText[i].text || m.fontSize !== currentText[i].fontSize || m.visible !== currentText[i].visible @@ -436,6 +439,8 @@ export const useCompositorLayers = ( m.width !== currentImg[i].width || m.height !== currentImg[i].height || m.opacity !== currentImg[i].opacity || + m.rotationDegrees !== currentImg[i].rotationDegrees || + m.zIndex !== currentImg[i].zIndex || m.visible !== currentImg[i].visible ); return changed ? merged : currentImg; @@ -476,8 +481,8 @@ export const useCompositorLayers = ( width: imgOverlay.width, height: imgOverlay.height, opacity: imgOverlay.opacity, - zIndex: 0, - rotationDegrees: 0, + zIndex: imgOverlay.zIndex, + rotationDegrees: imgOverlay.rotationDegrees, visible: imgOverlay.visible, }, kind: 'image', @@ -632,11 +637,14 @@ export const useCompositorLayers = ( newY = orig.y + (orig.height - newH); } - // Constrain video layer resize to maintain aspect ratio. - // Text/image overlays (layerKind !== 'video') are not constrained here - // because text overlays no longer have resize handles and image overlays - // may intentionally use different dimensions. - if (state.layerKind === 'video' && orig.width > 0 && orig.height > 0) { + // Constrain video and image layer resize to maintain aspect ratio. + // Text overlays are not constrained here because they no longer have + // resize handles. + if ( + (state.layerKind === 'video' || state.layerKind === 'image') && + orig.width > 0 && + orig.height > 0 + ) { const ar = orig.width / orig.height; const isCorner = handle.length === 2; // 'ne', 'nw', 'se', 'sw' @@ -999,8 +1007,18 @@ export const useCompositorLayers = ( // ── Image overlay CRUD ──────────────────────────────────────────────────── const addImageOverlay = useCallback( - (dataBase64: string) => { + (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => { setImageOverlays((prev) => { + // Compute initial rect that preserves source aspect ratio. + // Fit the image within a 200px box on its largest side. + const maxDim = 200; + let w = maxDim; + let h = maxDim; + if (naturalWidth && naturalHeight && naturalWidth > 0 && naturalHeight > 0) { + const scale = Math.min(maxDim / naturalWidth, maxDim / naturalHeight, 1); + w = Math.max(1, Math.round(naturalWidth * scale)); + h = Math.max(1, Math.round(naturalHeight * scale)); + } const next: ImageOverlayState[] = [ ...prev, { @@ -1008,11 +1026,12 @@ export const useCompositorLayers = ( dataBase64, x: 40, y: 40 + prev.length * 60, - width: 200, - height: 200, + width: w, + height: h, opacity: 1.0, rotationDegrees: 0, - zIndex: 0, + // Z-index band: image overlays use 200+ (video: 0–99, text: 100–199) + zIndex: 200 + prev.length, visible: true, }, ]; diff --git a/ui/src/nodes/CompositorNode.tsx b/ui/src/nodes/CompositorNode.tsx index 76ff6ccc..9c899c70 100644 --- a/ui/src/nodes/CompositorNode.tsx +++ b/ui/src/nodes/CompositorNode.tsx @@ -435,7 +435,7 @@ const OverlayList: React.FC<{ onAddText: (text: string) => void; onUpdateText: (id: string, updates: Partial>) => void; onRemoveText: (id: string) => void; - onAddImage: (dataBase64: string) => void; + onAddImage: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; onRemoveImage: (id: string) => void; disabled: boolean; }> = React.memo( @@ -476,7 +476,13 @@ const OverlayList: React.FC<{ const result = reader.result as string; // Strip the data:image/...;base64, prefix const base64 = result.split(',')[1]; - if (base64) onAddImage(base64); + if (!base64) return; + // Detect natural dimensions so the initial rect preserves + // the source aspect ratio. + const img = new window.Image(); + img.onload = () => onAddImage(base64, img.naturalWidth, img.naturalHeight); + img.onerror = () => onAddImage(base64); + img.src = result; }; reader.readAsDataURL(file); e.target.value = ''; @@ -659,7 +665,8 @@ const UnifiedLayerList: React.FC<{ onAddText: (text: string) => void; onUpdateText: (id: string, updates: Partial>) => void; onRemoveText: (id: string) => void; - onAddImage: (dataBase64: string) => void; + onAddImage: (dataBase64: string, naturalWidth?: number, naturalHeight?: number) => void; + onUpdateImage: (id: string, updates: Partial>) => void; onRemoveImage: (id: string) => void; disabled: boolean; }> = React.memo( @@ -677,6 +684,7 @@ const UnifiedLayerList: React.FC<{ onUpdateText, onRemoveText, onAddImage, + onUpdateImage, onRemoveImage, disabled, }) => { @@ -727,8 +735,9 @@ const UnifiedLayerList: React.FC<{ const [menuOpen, setMenuOpen] = useState(false); const fileInputRef = useRef(null); - // Find selected text overlay for inline editing controls + // Find selected text/image overlay for inline editing controls const selectedTextOverlay = textOverlays.find((o) => o.id === selectedLayerId); + const selectedImageOverlay = imageOverlays.find((o) => o.id === selectedLayerId); const handleAddText = useCallback(() => { onAddText('Text'); @@ -750,7 +759,13 @@ const UnifiedLayerList: React.FC<{ reader.onload = () => { const result = reader.result as string; const base64 = result.split(',')[1]; - if (base64) onAddImage(base64); + if (!base64) return; + // Detect natural dimensions so the initial rect preserves + // the source aspect ratio. + const img = new window.Image(); + img.onload = () => onAddImage(base64, img.naturalWidth, img.naturalHeight); + img.onerror = () => onAddImage(base64); + img.src = result; }; reader.readAsDataURL(file); e.target.value = ''; @@ -1020,6 +1035,58 @@ const UnifiedLayerList: React.FC<{ )} + + {/* Controls for the selected image overlay */} + {selectedImageOverlay && ( + <> + + Image #{imageOverlays.indexOf(selectedImageOverlay)} + + ({Math.round(selectedImageOverlay.x)}, {Math.round(selectedImageOverlay.y)}) + + + + + Opacity + + onUpdateImage(selectedImageOverlay.id, { + opacity: Number.parseFloat(e.target.value), + }) + } + disabled={disabled} + className="nodrag nopan" + /> + {(selectedImageOverlay.opacity * 100).toFixed(0)}% + + + + Rotation + + onUpdateImage(selectedImageOverlay.id, { + rotationDegrees: Number.parseFloat(e.target.value), + }) + } + disabled={disabled} + className="nodrag nopan" + /> + {selectedImageOverlay.rotationDegrees.toFixed(0)}° + + + )} ); } @@ -1051,6 +1118,7 @@ const CompositorNode: React.FC = React.memo(({ id, data, se updateTextOverlay, removeTextOverlay, addImageOverlay, + updateImageOverlay, removeImageOverlay, } = useCompositorLayers({ nodeId: id, @@ -1136,6 +1204,7 @@ const CompositorNode: React.FC = React.memo(({ id, data, se onUpdateText={updateTextOverlay} onRemoveText={removeTextOverlay} onAddImage={addImageOverlay} + onUpdateImage={updateImageOverlay} onRemoveImage={removeImageOverlay} disabled={disabled} />