From e53c8acb00296dd474353b0f33aa1d93329c8679 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 22 Apr 2025 18:04:59 -0700 Subject: [PATCH 1/3] More work on reset. --- web-codecs/Cargo.toml | 4 + web-codecs/src/audio/data.rs | 190 +++++++++++++++++++++++++++++--- web-codecs/src/audio/encoder.rs | 2 +- web-codecs/src/error.rs | 3 + web-codecs/src/video/encoder.rs | 2 +- web-codecs/src/video/frame.rs | 34 ++++-- web-message/Cargo.toml | 2 +- web-message/src/error.rs | 2 +- web-message/src/message.rs | 36 +++++- 9 files changed, 241 insertions(+), 34 deletions(-) diff --git a/web-codecs/Cargo.toml b/web-codecs/Cargo.toml index 6261dc9..75eb6f2 100644 --- a/web-codecs/Cargo.toml +++ b/web-codecs/Cargo.toml @@ -12,6 +12,7 @@ categories = ["wasm", "multimedia", "web-programming", "api-bindings"] rust-version = "1.85" [dependencies] +bytemuck = "1.22" bytes = "1" derive_more = { version = "2", features = ["from", "display"] } js-sys = "0.3.77" @@ -57,4 +58,7 @@ features = [ "AudioEncoderInit", "AudioEncoderConfig", "AudioSampleFormat", + "AudioDataCopyToOptions", + "AudioDataInit", + "console", ] diff --git a/web-codecs/src/audio/data.rs b/web-codecs/src/audio/data.rs index 3f338e3..08edcf8 100644 --- a/web-codecs/src/audio/data.rs +++ b/web-codecs/src/audio/data.rs @@ -1,45 +1,201 @@ +use std::ops::{Deref, DerefMut}; use std::time::Duration; -use derive_more::From; +use crate::{Error, Result, Timestamp}; -use crate::Timestamp; +pub use web_sys::AudioSampleFormat as AudioDataFormat; -#[derive(Debug, From)] -pub struct AudioData(web_sys::AudioData); +/// A wrapper around [web_sys::AudioData] that closes on Drop. +// It's an option so `leak` can return the inner AudioData if needed. +#[derive(Debug)] +pub struct AudioData(Option); impl AudioData { + /// A helper to construct AudioData in a more type-safe way. + /// This currently only supports F32. + pub fn new<'a>( + channels: impl Iterator + ExactSizeIterator, + sample_rate: u32, + timestamp: Timestamp, + ) -> Result { + let mut channels = channels.enumerate(); + let channel_count = channels.size_hint().0; + let (_, channel) = channels.next().ok_or(Error::NoChannels)?; + + let frame_count = channel.len(); + let total_samples = channel_count * frame_count; + + // Annoyingly, we need to create a contiguous buffer for the data. + let data = js_sys::Float32Array::new_with_length(total_samples as _); + + // Copy the first channel using a Float32Array as a view into the buffer. + let slice = js_sys::Float32Array::new_with_byte_offset_and_length(&data.buffer(), 0, frame_count as _); + slice.copy_from(channel); + + for (i, channel) in channels { + // Copy the other channels using a Float32Array as a view into the buffer. + let slice = js_sys::Float32Array::new_with_byte_offset_and_length( + &data.buffer(), + (i * frame_count) as u32, + frame_count as _, + ); + slice.copy_from(channel); + } + + let init = web_sys::AudioDataInit::new( + &data, + AudioDataFormat::F32Planar, + channel_count as _, + frame_count as _, + sample_rate as _, + timestamp.as_micros() as _, + ); + + // Manually add `transfer` to the init options. + // TODO Update web_sys to support this natively. + // I'm not even sure if this works. + let transfer = js_sys::Array::new(); + transfer.push(&data.buffer()); + js_sys::Reflect::set(&init, &js_sys::JsString::from("transfer"), &transfer)?; + + let audio_data = web_sys::AudioData::new(&init)?; + Ok(Self(Some(audio_data))) + } + pub fn timestamp(&self) -> Timestamp { - Timestamp::from_micros(self.0.timestamp() as _) + Timestamp::from_micros(self.0.as_ref().unwrap().timestamp() as _) } pub fn duration(&self) -> Duration { - Duration::from_micros(self.0.duration() as _) + Duration::from_micros(self.0.as_ref().unwrap().duration() as _) } - pub fn format(&self) -> Option { - self.0.format() + pub fn sample_rate(&self) -> u32 { + self.0.as_ref().unwrap().sample_rate() as u32 } - pub fn sample_rate(&self) -> u32 { - self.0.sample_rate() as u32 + pub fn append_to(&self, dst: &mut T, channel: usize, options: AudioCopyOptions) -> Result<()> { + dst.append_to(self, channel, options) } - pub fn frame_count(&self) -> u32 { - self.0.number_of_frames() + pub fn copy_to(&self, dst: &mut T, channel: usize, options: AudioCopyOptions) -> Result<()> { + dst.copy_to(self, channel, options) } - pub fn channel_count(&self) -> u32 { - self.0.number_of_channels() + pub fn leak(mut self) -> web_sys::AudioData { + self.0.take().unwrap() } +} - pub fn inner(&self) -> &web_sys::AudioData { - &self.0 +impl Clone for AudioData { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Deref for AudioData { + type Target = web_sys::AudioData; + + fn deref(&self) -> &Self::Target { + self.0.as_ref().unwrap() + } +} + +impl DerefMut for AudioData { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.as_mut().unwrap() } } // Make sure we close the frame on drop. impl Drop for AudioData { fn drop(&mut self) { - self.0.close(); + if let Some(audio_data) = self.0.take() { + audio_data.close(); + } + } +} + +impl From for AudioData { + fn from(this: web_sys::AudioData) -> Self { + Self(Some(this)) + } +} + +pub trait AudioCopy { + fn copy_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()>; +} + +impl AudioCopy for [u8] { + fn copy_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()> { + let options = options.into_web_sys(channel); + // NOTE: The format is unuset so it will default to the AudioData format. + // This means you couldn't export as U8Planar for whatever that's worth... + data.0.as_ref().unwrap().copy_to_with_u8_slice(self, &options)?; + Ok(()) + } +} + +impl AudioCopy for [f32] { + fn copy_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()> { + let options = options.into_web_sys(channel); + options.set_format(AudioDataFormat::F32Planar); + + // Cast from a f32 to a u8 slice. + let bytes = bytemuck::cast_slice_mut(self); + data.0.as_ref().unwrap().copy_to_with_u8_slice(bytes, &options)?; + Ok(()) + } +} + +impl AudioCopy for js_sys::Uint8Array { + fn copy_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()> { + let options = options.into_web_sys(channel); + data.0.as_ref().unwrap().copy_to_with_u8_array(self, &options)?; + Ok(()) + } +} + +impl AudioCopy for js_sys::Float32Array { + fn copy_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()> { + let options = options.into_web_sys(channel); + data.0.as_ref().unwrap().copy_to_with_buffer_source(self, &options)?; + Ok(()) + } +} + +pub trait AudioAppend { + fn append_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()>; +} + +impl AudioAppend for Vec { + fn append_to(&mut self, data: &AudioData, channel: usize, options: AudioCopyOptions) -> Result<()> { + // TODO do unsafe stuff to avoid zeroing the buffer. + let grow = options.count.unwrap_or(data.number_of_frames() as _) - options.offset; + let offset = self.len(); + self.resize(offset + grow, 0.0); + + let options = options.into_web_sys(channel); + let bytes = bytemuck::cast_slice_mut(&mut self[offset..]); + data.0.as_ref().unwrap().copy_to_with_u8_slice(bytes, &options)?; + + Ok(()) + } +} + +#[derive(Debug, Default)] +pub struct AudioCopyOptions { + pub offset: usize, // defaults to 0 + pub count: Option, // defaults to remainder +} + +impl AudioCopyOptions { + fn into_web_sys(self, channel: usize) -> web_sys::AudioDataCopyToOptions { + let options = web_sys::AudioDataCopyToOptions::new(channel as _); + options.set_frame_offset(self.offset as _); + if let Some(count) = self.count { + options.set_frame_count(count as _); + } + options } } diff --git a/web-codecs/src/audio/encoder.rs b/web-codecs/src/audio/encoder.rs index 5ae80c4..1f0d096 100644 --- a/web-codecs/src/audio/encoder.rs +++ b/web-codecs/src/audio/encoder.rs @@ -127,7 +127,7 @@ impl AudioEncoder { } pub fn encode(&mut self, frame: &AudioData) -> Result<(), Error> { - self.inner.encode(frame.inner())?; + self.inner.encode(frame)?; Ok(()) } diff --git a/web-codecs/src/error.rs b/web-codecs/src/error.rs index 08a5b0d..254734b 100644 --- a/web-codecs/src/error.rs +++ b/web-codecs/src/error.rs @@ -8,6 +8,9 @@ pub enum Error { #[error("invalid dimensions")] InvalidDimensions, + #[error("no channels")] + NoChannels, + #[error("unknown error: {0:?}")] Unknown(JsValue), } diff --git a/web-codecs/src/video/encoder.rs b/web-codecs/src/video/encoder.rs index 317c02f..66d5dc5 100644 --- a/web-codecs/src/video/encoder.rs +++ b/web-codecs/src/video/encoder.rs @@ -238,7 +238,7 @@ impl VideoEncoder { *last_keyframe = Some(timestamp); } - self.inner.encode_with_options(frame.inner(), &o)?; + self.inner.encode_with_options(frame, &o)?; Ok(()) } diff --git a/web-codecs/src/video/frame.rs b/web-codecs/src/video/frame.rs index 6b8094b..c264532 100644 --- a/web-codecs/src/video/frame.rs +++ b/web-codecs/src/video/frame.rs @@ -1,4 +1,7 @@ -use std::time::Duration; +use std::{ + ops::{Deref, DerefMut}, + time::Duration, +}; use derive_more::From; @@ -15,25 +18,32 @@ impl VideoFrame { pub fn duration(&self) -> Option { Some(Duration::from_micros(self.0.duration()? as _)) } +} - pub fn display_width(&self) -> u32 { - self.0.display_width() +// Avoid closing the video frame on transfer by cloning it. +impl From for web_sys::VideoFrame { + fn from(this: VideoFrame) -> Self { + this.0.clone().expect("detached") } +} - pub fn display_height(&self) -> u32 { - self.0.display_height() +impl Clone for VideoFrame { + fn clone(&self) -> Self { + Self(self.0.clone().expect("detached")) } +} - pub fn coded_width(&self) -> u32 { - self.0.coded_width() - } +impl Deref for VideoFrame { + type Target = web_sys::VideoFrame; - pub fn coded_height(&self) -> u32 { - self.0.coded_height() + fn deref(&self) -> &Self::Target { + &self.0 } +} - pub fn inner(&self) -> &web_sys::VideoFrame { - &self.0 +impl DerefMut for VideoFrame { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 } } diff --git a/web-message/Cargo.toml b/web-message/Cargo.toml index 6569910..6929176 100644 --- a/web-message/Cargo.toml +++ b/web-message/Cargo.toml @@ -14,7 +14,7 @@ default = ["derive"] derive = ["dep:web-message-derive"] # These features implement the Message interface for popular crates: -url = ["dep:url"] +Url = ["dep:url"] # These feature names copy web_sys for all (currently) transferable types. # https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects diff --git a/web-message/src/error.rs b/web-message/src/error.rs index 3c2c582..24bee42 100644 --- a/web-message/src/error.rs +++ b/web-message/src/error.rs @@ -15,7 +15,7 @@ pub enum Error { #[error("unknown tag")] UnknownTag, - #[cfg(feature = "url")] + #[cfg(feature = "Url")] #[error("invalid URL: {0}")] InvalidUrl(url::ParseError), } diff --git a/web-message/src/message.rs b/web-message/src/message.rs index 1860b1d..7cc4ce3 100644 --- a/web-message/src/message.rs +++ b/web-message/src/message.rs @@ -113,6 +113,40 @@ impl Message for js_sys::ArrayBuffer { } } +macro_rules! typed_array { + ($($t:ident),*,) => { + $( + impl Message for js_sys::$t { + fn into_message(self, transferable: &mut Array) -> JsValue { + transferable.push(&self.buffer()); + self.into() + } + + fn from_message(message: JsValue) -> Result { + message + .dyn_into::() + .map_err(|_| Error::UnexpectedType) + } + } + )* + }; +} + +// These are all the types that wrap an ArrayBuffer and can be transferred. +typed_array!( + Float32Array, + Float64Array, + Int8Array, + Int16Array, + Int32Array, + Uint8Array, + Uint8ClampedArray, + Uint16Array, + Uint32Array, + BigInt64Array, + BigUint64Array, +); + macro_rules! transferable_feature { ($($feature:literal = $t:ident),* $(,)?) => { $( @@ -151,7 +185,7 @@ transferable_feature!( "MidiAccess" = MidiAccess, ); -#[cfg(feature = "url")] +#[cfg(feature = "Url")] impl Message for url::Url { fn into_message(self, _transferable: &mut Array) -> JsValue { self.to_string().into() From e2726205c126ce4131e80c740c8f274d42fec15f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 May 2025 09:44:44 -0700 Subject: [PATCH 2/3] Add a closed method. --- web-streams/src/reader.rs | 5 +++++ web-streams/src/writer.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/web-streams/src/reader.rs b/web-streams/src/reader.rs index 9bdbd66..03d9c12 100644 --- a/web-streams/src/reader.rs +++ b/web-streams/src/reader.rs @@ -51,6 +51,11 @@ impl Reader { let str = JsValue::from_str(reason); self.inner.cancel_with_reason(&str).ignore(); } + + pub async fn closed(&self) -> Result<(), Error> { + JsFuture::from(self.inner.closed()).await?; + Ok(()) + } } impl Drop for Reader { diff --git a/web-streams/src/writer.rs b/web-streams/src/writer.rs index 625250f..8f37976 100644 --- a/web-streams/src/writer.rs +++ b/web-streams/src/writer.rs @@ -28,6 +28,11 @@ impl Writer { let str = JsValue::from_str(reason); self.inner.abort_with_reason(&str).ignore(); } + + pub async fn closed(&self) -> Result<(), Error> { + JsFuture::from(self.inner.closed()).await?; + Ok(()) + } } impl Drop for Writer { From e30895be2c88aa728a9d5824d7da79ce3a4c60e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 May 2025 10:01:34 -0700 Subject: [PATCH 3/3] clippy --- web-codecs/src/audio/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-codecs/src/audio/data.rs b/web-codecs/src/audio/data.rs index 08edcf8..76f8859 100644 --- a/web-codecs/src/audio/data.rs +++ b/web-codecs/src/audio/data.rs @@ -14,7 +14,7 @@ impl AudioData { /// A helper to construct AudioData in a more type-safe way. /// This currently only supports F32. pub fn new<'a>( - channels: impl Iterator + ExactSizeIterator, + channels: impl ExactSizeIterator, sample_rate: u32, timestamp: Timestamp, ) -> Result {