From 8c61100f69b4b96efc4af4b71ae2b9ee330fa378 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:12:11 +0100 Subject: [PATCH 01/27] feat(core): use a pool for frame to lower memory allocation --- .../streampack/core/elements/data/Frame.kt | 88 ++++++++---------- .../encoders/mediacodec/MediaCodecEncoder.kt | 41 +++++---- .../elements/endpoints/CombineEndpoint.kt | 2 +- .../muxers/mp4/models/TrackChunks.kt | 1 + .../endpoints/composites/muxers/ts/TsMuxer.kt | 19 ++-- .../core/elements/utils/pool/FramePool.kt | 57 ++++++++++++ .../{IFrameFactory.kt => IRawFrameFactory.kt} | 0 .../core/elements/utils/pool/ObjectPool.kt | 89 +++++++++++++++++++ .../core/elements/utils/FakeFrames.kt | 3 +- 9 files changed, 224 insertions(+), 76 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt rename core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/{IFrameFactory.kt => IRawFrameFactory.kt} (100%) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 018f4dbc1..14e337598 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,7 +16,6 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes import java.io.Closeable import java.nio.ByteBuffer @@ -48,111 +47,102 @@ data class RawFrame( } } - -data class Frame( +/** + * Encoded frame representation + */ +interface Frame { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - val ptsInUs: Long, + val ptsInUs: Long /** * Decoded timestamp in µs (not used). */ - val dtsInUs: Long? = null, + val dtsInUs: Long? /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - val isKeyFrame: Boolean, + val isKeyFrame: Boolean /** * Contains csd buffers for key frames and audio frames only. * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List?, + val extra: List? /** * Contains frame format.. * TODO: to remove */ val format: MediaFormat -) { - init { - removePrefixes() - } } -/** - * Removes the [Frame.extra] prefixes from the [Frame.rawBuffer]. - * - * With MediaCodec, the encoded frames may contain prefixes like SPS, PPS for H264/H265 key frames. - * It also modifies the position of the [Frame.rawBuffer] to skip the prefixes.s - * - * @return A [ByteBuffer] without prefixes. - */ -fun Frame.removePrefixes(): ByteBuffer { - return if (extra != null) { - rawBuffer.removePrefixes(extra) - } else { - rawBuffer - } +fun Frame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + ptsInUs: Long = this.ptsInUs, + dtsInUs: Long? = this.dtsInUs, + isKeyFrame: Boolean = this.isKeyFrame, + extra: List? = this.extra, + format: MediaFormat = this.format +): Frame { + return MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format + ) } -fun FrameWithCloseable( +/** + * A mutable [Frame] internal representation. + * + * The purpose is to get reusable [Frame] + */ +data class MutableFrame( /** * Contains an audio or video frame data. */ - rawBuffer: ByteBuffer, + override var rawBuffer: ByteBuffer, /** * Presentation timestamp in µs */ - ptsInUs: Long, + override var ptsInUs: Long, /** * Decoded timestamp in µs (not used). */ - dtsInUs: Long?, + override var dtsInUs: Long?, /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - isKeyFrame: Boolean, + override var isKeyFrame: Boolean, /** * Contains csd buffers for key frames and audio frames only. * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - extra: List?, + override var extra: List?, /** * Contains frame format.. + * TODO: to remove */ - format: MediaFormat, - - /** - * A callback to call when frame is closed. - */ - onClosed: (FrameWithCloseable) -> Unit, -) = FrameWithCloseable( - Frame( - rawBuffer, - ptsInUs, - dtsInUs, - isKeyFrame, - extra, - format - ), - onClosed -) + override var format: MediaFormat +) : Frame /** * Frame internal representation. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index e62806ca5..1b90d17b9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -31,6 +31,8 @@ import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extens import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extensions.isValid import io.github.thibaultbee.streampack.core.elements.utils.extensions.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.put +import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -40,6 +42,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import java.io.Closeable import kotlin.math.min /** @@ -246,6 +249,8 @@ internal constructor( if (input is SurfaceInput) { input.release() } + + frameFactory.close() } catch (_: Throwable) { } finally { setState(State.RELEASED) @@ -587,9 +592,11 @@ internal constructor( class FrameFactory( private val codec: MediaCodec, private val isVideo: Boolean - ) { + ) : Closeable { private var previousPresentationTimeUs = 0L + private val pool = FramePool() + /** * Create a [Frame] from a [RawFrame] * @@ -623,30 +630,34 @@ internal constructor( tag: String ): FrameWithCloseable { val buffer = requireNotNull(codec.getOutputBuffer(index)) + val extra = if (isKeyFrame || !isVideo) { + outputFormat.extra + } else { + null + } + val rawBuffer = if (extra != null) { + buffer.removePrefixes(extra) + } else { + buffer + } + + val frame = pool.get(rawBuffer, ptsInUs, null, isKeyFrame, extra, outputFormat) + return FrameWithCloseable( - buffer, - ptsInUs, // pts - null, // dts - isKeyFrame, - try { - if (isKeyFrame || !isVideo) { - outputFormat.extra - } else { - null - } - } catch (_: Throwable) { - null - }, - outputFormat, + frame, onClosed = { try { codec.releaseOutputBuffer(index, false) + pool.put(frame) } catch (t: Throwable) { Logger.w(tag, "Failed to release output buffer for code: ${t.message}") } }) } + override fun close() { + pool.close() + } } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 7d7d8f721..4802f24e1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect import io.github.thibaultbee.streampack.core.logger.Logger @@ -36,7 +37,6 @@ import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch - /** * Combines multiple endpoints into one. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 076d1c132..dca110e4f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import android.util.Size import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index d0ab92de7..f38231870 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -19,6 +19,7 @@ import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal @@ -86,16 +87,15 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_AVC -> { // Copy sps & pps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for AVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.forEach { buffer.put(it) } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -107,17 +107,16 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_HEVC -> { // Copy sps & pps & vps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for HEVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.forEach { buffer.put(it) } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt new file mode 100644 index 000000000..60281eeff --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame +import java.nio.ByteBuffer +import java.util.ArrayDeque + + +/** + * A pool of [MutableFrame]. + */ +internal class FramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + ptsInUs: Long, + dtsInUs: Long?, + isKeyFrame: Boolean, + extra: List?, + format: MediaFormat + ): MutableFrame { + val frame = get() + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.ptsInUs = ptsInUs + frame.dtsInUs = dtsInUs + frame.isKeyFrame = isKeyFrame + frame.extra = extra + frame.format = format + frame + } else { + MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format + ) + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt similarity index 100% rename from core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt new file mode 100644 index 000000000..b405aa274 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import java.io.Closeable +import java.util.ArrayDeque +import java.util.concurrent.atomic.AtomicBoolean + +/** + * A pool of objects. + * + * The pool is thread-safe. + * + * The implementation is required to add a `get` methods. + * + * @param T the type of object to pool + */ +internal sealed class ObjectPool() : Closeable { + private val pool = ArrayDeque() + + private val isClosed = AtomicBoolean(false) + + protected fun get(): T? { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + + return synchronized(pool) { + if (!pool.isEmpty()) { + pool.removeFirst() + } else { + null + } + } + } + + /** + * Puts an object in the pool. + * + * @param any the object to put + */ + fun put(any: T) { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + synchronized(pool) { + pool.addLast(any) + } + } + + /** + * Clears the pool. + */ + fun clear() { + if (isClosed.get()) { + return + } + synchronized(pool) { + pool.clear() + } + } + + /** + * Closes the pool. + * + * After a pool is closed, it cannot be used anymore. + */ + override fun close() { + if (isClosed.getAndSet(true)) { + return + } + synchronized(pool) { + pool.clear() + } + } +} \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index 83139d45c..4b1a4c94b 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random @@ -51,7 +52,7 @@ object FakeFrames { ) ) } - return Frame( + return MutableFrame( buffer, pts, dts, From 20b0435dfd55edd4991c2e777c090112020a961a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:15:02 +0100 Subject: [PATCH 02/27] refactor(*): merge FrameWithCloseable with MutableFrame to avoid memory allocation --- .../core/elements/endpoints/DummyEndpoint.kt | 6 ++-- .../streampack/core/elements/data/Frame.kt | 34 ++++++++----------- .../core/elements/encoders/IEncoder.kt | 4 +-- .../encoders/mediacodec/MediaCodecEncoder.kt | 21 ++++++------ .../elements/endpoints/CombineEndpoint.kt | 14 ++++---- .../elements/endpoints/DynamicEndpoint.kt | 6 ++-- .../core/elements/endpoints/IEndpoint.kt | 9 +++-- .../elements/endpoints/MediaMuxerEndpoint.kt | 7 ++-- .../endpoints/composites/CompositeEndpoint.kt | 6 ++-- .../composites/muxers/IMuxerInternal.kt | 4 +-- .../composites/muxers/mp4/Mp4Muxer.kt | 7 ++-- .../endpoints/composites/muxers/ts/TsMuxer.kt | 8 ++--- .../core/elements/utils/pool/FramePool.kt | 8 +++-- .../core/elements/utils/pool/ObjectPool.kt | 1 + .../encoding/EncodingPipelineOutput.kt | 14 ++++---- .../elements/endpoints/DynamicEndpointTest.kt | 4 +-- .../composites/muxers/ts/TsMuxerTest.kt | 14 ++++---- .../core/elements/utils/FakeFrames.kt | 20 ----------- .../flv/elements/endpoints/FlvEndpoints.kt | 7 ++-- .../composites/muxer/utils/FlvTagBuilder.kt | 12 +++---- .../rtmp/elements/endpoints/RtmpEndpoint.kt | 7 ++-- 21 files changed, 91 insertions(+), 122 deletions(-) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index b4a06c464..bf8caea0b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -19,7 +19,6 @@ import android.content.Context import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow @@ -58,11 +57,10 @@ class DummyEndpoint : IEndpointInternal { _isOpenFlow.emit(false) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { Log.i(TAG, "write: $frame") _frameFlow.emit(frame) - closeableFrame.close() + frame.close() } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 14e337598..6d293060c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -50,7 +50,7 @@ data class RawFrame( /** * Encoded frame representation */ -interface Frame { +interface Frame : Closeable { /** * Contains an audio or video frame data. */ @@ -85,13 +85,18 @@ interface Frame { val format: MediaFormat } +interface WithClosable { + val onClosed: (T) -> Unit +} + fun Frame.copy( rawBuffer: ByteBuffer = this.rawBuffer, ptsInUs: Long = this.ptsInUs, dtsInUs: Long? = this.dtsInUs, isKeyFrame: Boolean = this.isKeyFrame, extra: List? = this.extra, - format: MediaFormat = this.format + format: MediaFormat = this.format, + onClosed: (Frame) -> Unit = {} ): Frame { return MutableFrame( rawBuffer = rawBuffer, @@ -99,7 +104,8 @@ fun Frame.copy( dtsInUs = dtsInUs, isKeyFrame = isKeyFrame, extra = extra, - format = format + format = format, + onClosed = onClosed ) } @@ -141,16 +147,13 @@ data class MutableFrame( * Contains frame format.. * TODO: to remove */ - override var format: MediaFormat -) : Frame + override var format: MediaFormat, -/** - * Frame internal representation. - */ -data class FrameWithCloseable( - val frame: Frame, - val onClosed: (FrameWithCloseable) -> Unit -) : Closeable { + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableFrame) -> Unit = {} +) : Frame, WithClosable { override fun close() { try { onClosed(this) @@ -159,10 +162,3 @@ data class FrameWithCloseable( } } } - -/** - * Uses the resource and unwraps the [Frame] to pass it to the given block. - */ -inline fun FrameWithCloseable.useAndUnwrap(block: (Frame) -> T) = use { - block(frame) -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt index 2bedd128a..b5d480a15 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt @@ -16,7 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.encoders import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendReleasable import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendStreamable @@ -77,7 +77,7 @@ interface IEncoderInternal : SuspendStreamable, SuspendReleasable, /** * A channel where the encoder will send encoded frames. */ - val outputChannel: SendChannel + val outputChannel: SendChannel } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 1b90d17b9..80ec9f025 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,7 +22,7 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode import io.github.thibaultbee.streampack.core.elements.encoders.IEncoderInternal @@ -249,8 +249,6 @@ internal constructor( if (input is SurfaceInput) { input.release() } - - frameFactory.close() } catch (_: Throwable) { } finally { setState(State.RELEASED) @@ -604,7 +602,7 @@ internal constructor( */ fun frame( index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String - ): FrameWithCloseable { + ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { pts = previousPresentationTimeUs + 1 @@ -628,7 +626,7 @@ internal constructor( ptsInUs: Long, isKeyFrame: Boolean, tag: String - ): FrameWithCloseable { + ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { outputFormat.extra @@ -641,11 +639,14 @@ internal constructor( buffer } - val frame = pool.get(rawBuffer, ptsInUs, null, isKeyFrame, extra, outputFormat) - - return FrameWithCloseable( - frame, - onClosed = { + return pool.get( + rawBuffer, + ptsInUs, + null, + isKeyFrame, + extra, + outputFormat, + onClosed = { frame -> try { codec.releaseOutputBuffer(index, false) pool.put(frame) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 4802f24e1..f9d235323 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect @@ -218,8 +218,7 @@ open class CombineEndpoint( * * If all endpoints write fails, it throws the exception of the first endpoint that failed. */ - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { val throwables = mutableListOf() /** @@ -231,9 +230,10 @@ open class CombineEndpoint( endpointInternals.filter { it.isOpenFlow.value }.forEach { endpoint -> try { val deferred = CompletableDeferred() - val duplicatedFrame = FrameWithCloseable( - frame.copy(rawBuffer = frame.rawBuffer.duplicate()), - { deferred.complete(Unit) }) + val duplicatedFrame = frame.copy( + rawBuffer = frame.rawBuffer.duplicate(), + onClosed = { deferred.complete(Unit) } + ) val endpointStreamId = endpointsToStreamIdsMap[Pair(endpoint, streamPid)]!! deferreds += deferred @@ -246,7 +246,7 @@ open class CombineEndpoint( coroutineScope.launch { deferreds.forEach { it.await() } - closeableFrame.close() + frame.close() } if (throwables.isNotEmpty()) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt index c242000ef..1951fabc0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.createDefaultTsServiceInfo -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer @@ -146,8 +146,8 @@ open class DynamicEndpoint( endpoint.addStream(streamConfig) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) = - safeEndpoint { endpoint -> endpoint.write(closeableFrame, streamPid) } + override suspend fun write(frame: Frame, streamPid: Int) = + safeEndpoint { endpoint -> endpoint.write(frame, streamPid) } override suspend fun startStream() = safeEndpoint { endpoint -> endpoint.startStream() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt index b87e8627f..715c7d5c8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.FileSink import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendCloseable @@ -46,14 +45,14 @@ interface IEndpointInternal : IEndpoint, SuspendStreamable, /** * Writes a [Frame] to the [IEndpointInternal]. * - * The [FrameWithCloseable.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. - * The [IEndpointInternal] must called [FrameWithCloseable.close] even if the frame is dropped or it somehow crashes. + * The [Frame.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. + * The [IEndpointInternal] must called [Frame.close] even if the frame is dropped or it somehow crashes. * - * @param closeableFrame the [Frame] to write + * @param frame the [Frame] to write * @param streamPid the stream id the [Frame] belongs to */ suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int, ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt index 1d30fba65..cedb2462a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt @@ -24,7 +24,7 @@ import android.media.MediaMuxer.OutputFormat import android.os.Build import android.os.ParcelFileDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider @@ -142,9 +142,8 @@ class MediaMuxerEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) = withContext(ioDispatcher) { - val frame = closeableFrame.frame mutex.withLock { try { if (state != State.STARTED && state != State.PENDING_START) { @@ -183,7 +182,7 @@ class MediaMuxerEndpoint( Logger.e(TAG, "Error while writing frame: ${t.message}") throw t } finally { - closeableFrame.close() + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt index 92f58fcee..e85726a65 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -80,9 +80,9 @@ class CompositeEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int - ) = muxer.write(closeableFrame, streamPid) + ) = muxer.write(frame, streamPid) override suspend fun addStreams(streamConfigs: List): Map { return mutex.withLock { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt index 1b609738b..108736b7c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt @@ -15,7 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable @@ -32,7 +32,7 @@ interface IMuxerInternal : fun onOutputFrame(packet: Packet) } - fun write(closeableFrame: FrameWithCloseable, streamPid: Int) + fun write(frame: Frame, streamPid: Int) fun addStreams(streamsConfig: List): Map diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt index 4c4f630e5..b96ca44b2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt @@ -15,9 +15,8 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4 -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet -import io.github.thibaultbee.streampack.core.elements.data.useAndUnwrap import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.FileTypeBox @@ -58,9 +57,9 @@ class Mp4Muxer( override val streamConfigs: List get() = tracks.map { it.config } - override fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { + override fun write(frame: Frame, streamPid: Int) { synchronized(this) { - closeableFrame.useAndUnwrap { frame -> + frame.use { frame -> if (segmenter!!.mustWriteSegment(frame)) { writeSegment() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index f38231870..626eaebf8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -73,13 +72,12 @@ class TsMuxer : IMuxerInternal { /** * Encodes a frame to MPEG-TS format. * Each audio frames and each video key frames must come with an extra buffer containing sps, pps,... - * @param closeableFrame frame to mux + * @param frame frame to mux * @param streamPid Pid of frame stream. Throw a NoSuchElementException if streamPid refers to an unknown stream */ override fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame try { val pes = getPes(streamPid.toShort()) val mimeType = pes.stream.config.mimeType @@ -160,7 +158,7 @@ class TsMuxer : IMuxerInternal { generateStreams(newFrame, pes) } } finally { - closeableFrame.close() + frame.close() } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 60281eeff..112b0d73a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.utils.pool import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer -import java.util.ArrayDeque /** @@ -31,7 +30,8 @@ internal class FramePool() : ObjectPool() { dtsInUs: Long?, isKeyFrame: Boolean, extra: List?, - format: MediaFormat + format: MediaFormat, + onClosed: (MutableFrame) -> Unit ): MutableFrame { val frame = get() @@ -42,6 +42,7 @@ internal class FramePool() : ObjectPool() { frame.isKeyFrame = isKeyFrame frame.extra = extra frame.format = format + frame.onClosed = onClosed frame } else { MutableFrame( @@ -50,7 +51,8 @@ internal class FramePool() : ObjectPool() { dtsInUs = dtsInUs, isKeyFrame = isKeyFrame, extra = extra, - format = format + format = format, + onClosed = onClosed ) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt index b405aa274..73bc0b337 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.pool +import io.github.thibaultbee.streampack.core.logger.Logger import java.io.Closeable import java.util.ArrayDeque import java.util.concurrent.atomic.AtomicBoolean diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index cedbad83b..d3b8a2a05 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.pipelines.outputs.encoding import android.content.Context import android.view.Surface import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -239,7 +239,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -250,7 +250,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -259,10 +259,10 @@ internal class EncodingPipelineOutput( if (withAudio) { coroutineScope.launch(audioOutputDispatcher) { // Audio - audioEncoderListener.outputChannel.consumeEach { closeableFrame -> + audioEncoderListener.outputChannel.consumeEach { frame -> try { audioStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Audio frame received but audio stream is not set") } catch (t: Throwable) { onInternalError(t) @@ -273,10 +273,10 @@ internal class EncodingPipelineOutput( if (withVideo) { coroutineScope.launch(videoOutputDispatcher) { // Video - videoEncoderListener.outputChannel.consumeEach { closeableFrame -> + videoEncoderListener.outputChannel.consumeEach { frame -> try { videoStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Video frame received but video stream is not set") } catch (t: Throwable) { onInternalError(t) diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt index 2b8e3428b..3b96e15f9 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt @@ -6,7 +6,7 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.utils.DescriptorUtils -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.test.runTest import org.junit.Assert.assertFalse @@ -85,7 +85,7 @@ class DynamicEndpointTest { val dynamicEndpoint = DynamicEndpoint(context, Dispatchers.Default, Dispatchers.IO) try { dynamicEndpoint.write( - FakeFramesWithCloseable.create(MediaFormat.MIMETYPE_AUDIO_AAC), + FakeFrames.create(MediaFormat.MIMETYPE_AUDIO_AAC), 0 ) fail("Throwable expected") diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt index 1a5b3dc8d..b3fa4dd23 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.TSConst import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.Utils.createFakeServiceInfo -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -193,7 +193,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -206,7 +206,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -231,11 +231,11 @@ class TsMuxerTest { tsMux.addStreams(service, listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) } @@ -251,10 +251,10 @@ class TsMuxerTest { tsMux.addStreams(createFakeServiceInfo(), listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index 4b1a4c94b..b31ab2b44 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -17,7 +17,6 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random @@ -66,22 +65,3 @@ object FakeFrames { ) } } - -object FakeFramesWithCloseable { - fun create( - mimeType: String, - buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), - pts: Long = Random.nextLong(), - dts: Long? = null, - isKeyFrame: Boolean = false - ) = FrameWithCloseable( - FakeFrames.create( - mimeType, - buffer, - pts, - dts, - isKeyFrame - ), - {/* Nothing to do */ } - ) -} \ No newline at end of file diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt index 91bf48b07..661315379 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.flv.encode import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.OnMetadata import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType @@ -126,12 +126,11 @@ sealed class FlvEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt index 14df85e5b..a450972d7 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.m import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.Metadata import io.github.komedia.komuxer.logger.KomuxerLogger -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -106,21 +106,19 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { } suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, ts: Int, streamPid: Int ) { if (ts < 0) { Logger.w( TAG, - "Negative timestamp $ts for frame ${closeableFrame.frame}. Frame will be dropped." + "Negative timestamp $ts for frame $frame. Frame will be dropped." ) - closeableFrame.close() + frame.close() return } - val frame = closeableFrame.frame - val flvDatas = when (streamPid) { AUDIO_STREAM_PID -> audioStream?.create(frame) ?: throw IllegalStateException("Audio stream not added") @@ -133,7 +131,7 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { flvDatas.forEachIndexed { index, flvData -> if (index == flvDatas.lastIndex) { // Pass the close callback on the last element - channel.send(FLVTag(ts, flvData), { closeableFrame.close() }) + channel.send(FLVTag(ts, flvData), { frame.close() }) } else { channel.send(FLVTag(ts, flvData)) } diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt index b0d3d8eea..eab0e0eb9 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.rtmp.client.RtmpClient import io.github.komedia.komuxer.rtmp.connect import io.github.komedia.komuxer.rtmp.messages.command.StreamPublishType import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.ClosedException import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint @@ -164,12 +164,11 @@ class RtmpEndpoint internal constructor( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { From d389b0d8fcf2f109c0f832ea5ed3aa4bd6ad0e16 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:06:28 +0100 Subject: [PATCH 03/27] chore(core): frame copy use frame from pool for memory optimization --- .../streampack/core/elements/data/Frame.kt | 22 ++++--- .../muxers/mp4/models/TrackChunks.kt | 58 ++++++++++--------- .../endpoints/composites/muxers/ts/TsMuxer.kt | 10 +++- .../core/elements/utils/pool/FramePool.kt | 7 +++ 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 6d293060c..e7505acd2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import java.io.Closeable import java.nio.ByteBuffer @@ -89,6 +90,11 @@ interface WithClosable { val onClosed: (T) -> Unit } +/** + * Copy a [Frame] to a new [Frame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ fun Frame.copy( rawBuffer: ByteBuffer = this.rawBuffer, ptsInUs: Long = this.ptsInUs, @@ -98,15 +104,13 @@ fun Frame.copy( format: MediaFormat = this.format, onClosed: (Frame) -> Unit = {} ): Frame { - return MutableFrame( - rawBuffer = rawBuffer, - ptsInUs = ptsInUs, - dtsInUs = dtsInUs, - isKeyFrame = isKeyFrame, - extra = extra, - format = format, - onClosed = onClosed - ) + val pool = FramePool.default + return pool.get( + rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, + { frame -> + pool.put(frame) + onClosed(frame) + }) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index dca110e4f..33cea1f13 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -56,7 +56,6 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.VPCodecConfigurationBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createHandlerBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createTypeMediaHeaderBox -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.AudioSpecificConfigDescriptor import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.ESDescriptor @@ -70,6 +69,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import java.nio.ByteBuffer /** @@ -184,34 +184,38 @@ class TrackChunks( fun write() { chunks.forEach { chunk -> chunk.writeTo { frame -> - when (track.config.mimeType) { - MediaFormat.MIMETYPE_VIDEO_HEVC, - MediaFormat.MIMETYPE_VIDEO_AVC -> { - if (frame.rawBuffer.isAnnexB) { - // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() - val sizeBuffer = ByteBuffer.allocate(4) - sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) - onNewSample(sizeBuffer) - onNewSample(noStartCodeBuffer) - } else if (frame.rawBuffer.isAvcc) { - onNewSample(frame.rawBuffer) - } else { - throw IllegalArgumentException( - "Unsupported buffer format: buffer start with 0x${ - frame.rawBuffer.get( - 0 - ).toString(16) - }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ - frame.rawBuffer.get(2).toString(16) - }, 0x${frame.rawBuffer.get(3).toString(16)}" - ) + try { + when (track.config.mimeType) { + MediaFormat.MIMETYPE_VIDEO_HEVC, + MediaFormat.MIMETYPE_VIDEO_AVC -> { + if (frame.rawBuffer.isAnnexB) { + // Replace start code with size (from Annex B to AVCC) + val noStartCodeBuffer = frame.rawBuffer.removeStartCode() + val sizeBuffer = ByteBuffer.allocate(4) + sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) + onNewSample(sizeBuffer) + onNewSample(noStartCodeBuffer) + } else if (frame.rawBuffer.isAvcc) { + onNewSample(frame.rawBuffer) + } else { + throw IllegalArgumentException( + "Unsupported buffer format: buffer start with 0x${ + frame.rawBuffer.get( + 0 + ).toString(16) + }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ + frame.rawBuffer.get(2).toString(16) + }, 0x${frame.rawBuffer.get(3).toString(16)}" + ) + } } - } - else -> { - onNewSample(frame.rawBuffer) - } // Nothing + else -> { + onNewSample(frame.rawBuffer) + } // Nothing + } + } finally { + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index 626eaebf8..ba17e1291 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -154,8 +154,14 @@ class TsMuxer : IMuxerInternal { else -> throw IllegalArgumentException("Unsupported mimeType $mimeType") } - synchronized(this) { - generateStreams(newFrame, pes) + try { + synchronized(this) { + generateStreams(newFrame, pes) + } + } finally { + if (frame != newFrame) { + newFrame.close() + } } } finally { frame.close() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 112b0d73a..264917486 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -56,4 +56,11 @@ internal class FramePool() : ObjectPool() { ) } } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { FramePool() } + } } \ No newline at end of file From 4afc99f6fc334afce77d0769ab2e286995dbff81 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:16:49 +0100 Subject: [PATCH 04/27] chore(core): codec: reusable extra for memory optimization --- .../encoders/mediacodec/MediaCodecEncoder.kt | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 80ec9f025..deb443df4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -43,6 +43,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.io.Closeable +import java.nio.ByteBuffer import kotlin.math.min /** @@ -63,6 +64,12 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat private var outputFormat: MediaFormat? = null + set(value) { + extra = value?.extra + field = value + } + private var extra: List? = null + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } private val isVideo = encoderConfig.isVideo @@ -353,7 +360,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, outputFormat!!, info, tag + index, extra, outputFormat!!, info, tag ) try { listener.outputChannel.send(frame) @@ -601,7 +608,11 @@ internal constructor( * @return the created frame */ fun frame( - index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String + index: Int, + extra: List?, + outputFormat: MediaFormat, + info: BufferInfo, + tag: String ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { @@ -609,7 +620,7 @@ internal constructor( Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, extra, outputFormat, pts, info.isKeyFrame, tag) } /** @@ -622,6 +633,7 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, + extra: List?, outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, @@ -629,7 +641,7 @@ internal constructor( ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { - outputFormat.extra + extra!!.map { it.duplicate() } } else { null } From 2e781ba04ea40bc4666a987094b1b08c0d3b2a28 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Sun, 25 Jan 2026 21:52:19 +0100 Subject: [PATCH 05/27] refactor(core): move dummy endpoint to main source for test purpose --- .../encoding/EncodingPipelineOutputTest.kt | 3 --- .../core/elements/endpoints/DummyEndpoint.kt | 18 +++--------------- 2 files changed, 3 insertions(+), 18 deletions(-) rename core/src/{androidTest => main}/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt (86%) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 1531e0257..917940c47 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -335,9 +335,6 @@ class EncodingPipelineOutputTest { Random.nextLong() ) ) - - // Wait for frame to be received - assertNotNull(dummyEndpoint.frameFlow.filterNotNull().first()) } companion object { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt similarity index 86% rename from core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index bf8caea0b..429b0a078 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -16,7 +16,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context -import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -25,19 +24,16 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +/** + * A dummy endpoint for testing. + */ class DummyEndpoint : IEndpointInternal { private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() - private val _frameFlow = MutableStateFlow(null) - val frameFlow = _frameFlow.asStateFlow() - private val _isStreamingFlow = MutableStateFlow(false) val isStreamingFlow = _isStreamingFlow.asStateFlow() - private val _configFlow = MutableStateFlow(null) - val configFlow = _configFlow.asStateFlow() - override val info: IEndpoint.IEndpointInfo get() = TODO("Not yet implemented") @@ -58,18 +54,14 @@ class DummyEndpoint : IEndpointInternal { } override suspend fun write(frame: Frame, streamPid: Int) { - Log.i(TAG, "write: $frame") - _frameFlow.emit(frame) frame.close() } override suspend fun addStreams(streamConfigs: List): Map { - streamConfigs.forEach { _configFlow.emit(it) } return streamConfigs.associateWith { it.hashCode() } } override suspend fun addStream(streamConfig: CodecConfig): Int { - _configFlow.emit(streamConfig) return streamConfig.hashCode() } @@ -80,10 +72,6 @@ class DummyEndpoint : IEndpointInternal { override suspend fun stopStream() { _isStreamingFlow.emit(false) } - - companion object { - private const val TAG = "DummyEndpoint" - } } /** From 9d254780167ac41cef92b13d2e8dc9684293875b Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 26 Jan 2026 22:23:54 +0100 Subject: [PATCH 06/27] refactor(core): avoid slice to skip start code to reduce memory allocation --- .../encoders/mediacodec/MediaCodecEncoder.kt | 2 +- .../muxers/mp4/models/TrackChunks.kt | 4 +- .../avc/AVCDecoderConfigurationRecord.kt | 21 ++- .../hevc/HEVCDecoderConfigurationRecord.kt | 4 +- .../utils/extensions/ByteBufferExtensions.kt | 154 ++++++++++++------ .../extensions/ByteBufferExtensionsKtTest.kt | 45 ++++- 6 files changed, 160 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index deb443df4..fe033fb96 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -531,7 +531,7 @@ internal constructor( } val inputBuffer = requireNotNull(mediaCodec.getInputBuffer(inputBufferId)) val size = min(frame.rawBuffer.remaining(), inputBuffer.remaining()) - inputBuffer.put(frame.rawBuffer, frame.rawBuffer.position(), size) + inputBuffer.put(frame.rawBuffer, 0, size) mediaCodec.queueInputBuffer( inputBufferId, 0, size, frame.timestampInUs, 0 ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 33cea1f13..cbccabd76 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -66,7 +66,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodec import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils @@ -190,7 +190,7 @@ class TrackChunks( MediaFormat.MIMETYPE_VIDEO_AVC -> { if (frame.rawBuffer.isAnnexB) { // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() + val noStartCodeBuffer = frame.rawBuffer.skipStartCode() val sizeBuffer = ByteBuffer.allocate(4) sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) onNewSample(sizeBuffer) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt index 75281c053..0b0c0fd3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt @@ -19,8 +19,8 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.buffer.ByteBuffer import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaFormat import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import java.nio.ByteBuffer @@ -33,8 +33,8 @@ data class AVCDecoderConfigurationRecord( private val sps: List, private val pps: List ) : ByteBufferWriter() { - private val spsNoStartCode: List = sps.map { it.removeStartCode() } - private val ppsNoStartCode: List = pps.map { it.removeStartCode() } + private val spsNoStartCode: List = sps.map { it.skipStartCode() } + private val ppsNoStartCode: List = pps.map { it.skipStartCode() } override val size: Int = getSize(spsNoStartCode, ppsNoStartCode) @@ -86,11 +86,13 @@ data class AVCDecoderConfigurationRecord( sps: List, pps: List ): AVCDecoderConfigurationRecord { - val spsNoStartCode = sps.map { it.removeStartCode() } - val ppsNoStartCode = pps.map { it.removeStartCode() } - val profileIdc: Byte = spsNoStartCode[0].get(1) - val profileCompatibility = spsNoStartCode[0].get(2) - val levelIdc = spsNoStartCode[0].get(3) + val spsNoStartCode = sps.map { it.skipStartCode() } + val ppsNoStartCode = pps.map { it.skipStartCode() } + val firstSpsNoStartCode = spsNoStartCode[0] + val firstSpsNoStartCodePosition = firstSpsNoStartCode.position() + val profileIdc: Byte = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 1) + val profileCompatibility = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 2) + val levelIdc = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 3) return AVCDecoderConfigurationRecord( profileIdc = profileIdc, profileCompatibility = profileCompatibility, @@ -112,7 +114,8 @@ data class AVCDecoderConfigurationRecord( size += 2 + it.remaining() - it.startCodeSize } val spsStartCodeSize = sps[0].startCodeSize - val profileIdc = sps[0].get(spsStartCodeSize + 1).toInt() + val spsPosition = sps[0].position() + val profileIdc = sps[0].get(spsPosition + spsStartCodeSize + 1).toInt() if ((profileIdc == 100) || (profileIdc == 110) || (profileIdc == 122) || (profileIdc == 144)) { size += 4 } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt index 28a11669f..7bd86d908 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt @@ -20,7 +20,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaForma import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putLong48 import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl import io.github.thibaultbee.streampack.core.elements.utils.extensions.shr import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize @@ -195,7 +195,7 @@ data class HEVCDecoderConfigurationRecord( } data class NalUnit(val type: Type, val data: ByteBuffer, val completeness: Boolean = true) { - val noStartCodeData: ByteBuffer = data.removeStartCode() + val noStartCodeData: ByteBuffer = data.skipStartCode() fun write(buffer: ByteBuffer) { buffer.put((completeness shl 7) or type.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 66da15b18..e4beee7ab 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -86,12 +86,14 @@ fun ByteBuffer.put3x3Matrix(matrix: IntArray) { matrix.forEach { putInt(it) } } -fun ByteBuffer.put(buffer: ByteBuffer, offset: Int, length: Int) { - val limit = buffer.limit() - buffer.position(offset) - buffer.limit(offset + length) - this.put(buffer) - buffer.limit(limit) +fun ByteBuffer.put(src: ByteBuffer, offset: Int, length: Int) { + val limit = src.limit() + if (offset != 0) { + src.position(src.position() + offset) + } + src.limit(src.position() + offset + length) + this.put(src) + src.limit(limit) } fun ByteBuffer.getString(size: Int = this.remaining()): String { @@ -111,22 +113,38 @@ fun ByteBuffer.getLong(isLittleEndian: Boolean): Long { return value } -fun ByteBuffer.indicesOf(prefix: ByteArray): List { - if (prefix.isEmpty()) { - return emptyList() +/** + * Finds all occurrences of the given [needle] byte array within the ByteBuffer. + * @param needle The byte array sequence to search for. + * @return A list of starting indices for every match found. + */ +fun ByteBuffer.indicesOf(needle: ByteArray): List { + if (needle.isEmpty()) return emptyList() + + val results = mutableListOf() + val end = limit() - needle.size + var i = position() + + while (i <= end) { + if (match(i, needle)) { + results.add(i) + // Move forward by the needle's length to find the next non-overlapping match + // Or use i++ if you want to allow overlapping matches (e.g., "AAA" in "AAAA") + i += needle.size + } else { + i++ + } } + return results +} - val indices = mutableListOf() - - outer@ for (i in 0 until this.limit() - prefix.size + 1) { - for (j in prefix.indices) { - if (this.get(i + j) != prefix[j]) { - continue@outer - } +private fun ByteBuffer.match(start: Int, needle: ByteArray): Boolean { + for (idx in needle.indices) { + if (get(start + idx) != needle[idx]) { + return false } - indices.add(i) } - return indices + return true } /** @@ -137,18 +155,18 @@ fun ByteBuffer.slices(prefix: ByteArray): List { // Get all occurrence of prefix in buffer val indexes = this.indicesOf(prefix) + // Get slices indexes.forEachIndexed { index, i -> val nextPosition = if (indexes.indices.contains(index + 1)) { - indexes[index + 1] - 1 + indexes[index + 1] } else { - this.limit() - 1 + this.limit() } slices.add(Pair(i, nextPosition)) } - val array = this.array() return slices.map { - ByteBuffer.wrap(array.sliceArray(IntRange(it.first, it.second))) + this.slice(from = it.first, to = it.second) } } @@ -258,51 +276,72 @@ fun ByteBuffer.clone(): ByteBuffer { /** - * Get start code size of [ByteBuffer]. + * Get start code size of [ByteBuffer] from the current position */ val ByteBuffer.startCodeSize: Int - get() { - return if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x00.toByte() && this.get(3) == 0x01.toByte() - ) { - 4 - } else if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x01.toByte() - ) { - 3 - } else { - 0 - } + get() = getStartCodeSize(this.position()) + +/** + * Get start code size of [ByteBuffer] from the given [position]. + */ +fun ByteBuffer.getStartCodeSize( + position: Int +): Int { + return if (remaining() >= 4 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x00.toByte() && this.get(position + 3) == 0x01.toByte() + ) { + 4 + } else if (remaining() >= 3 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x01.toByte() + ) { + 3 + } else { + 0 } +} -fun ByteBuffer.removeStartCode(): ByteBuffer { +/** + * Moves the position after the start code. + */ +fun ByteBuffer.skipStartCode(): ByteBuffer { val startCodeSize = this.startCodeSize - this.position(startCodeSize) - return this.slice() + if (startCodeSize > 0) { + this.position(this.position() + startCodeSize) + } + return this } +private val emulationPreventionThreeByte = byteArrayOf(0x00, 0x00, 0x03) + +/** + * Removes all emulation prevention three bytes from [ByteBuffer]. + * + * @param headerLength [Int] of the header length before writing the [ByteBuffer]. + * @return [ByteBuffer] without emulation prevention three bytes + */ fun ByteBuffer.extractRbsp(headerLength: Int): ByteBuffer { - val rbsp = ByteBuffer.allocateDirect(this.remaining()) + val indices = this.indicesOf(emulationPreventionThreeByte) - val indices = this.indicesOf(byteArrayOf(0x00, 0x00, 0x03)) + val rbspSize = + this.remaining() - indices.size // remove 0x3 bytes for each emulation prevention three bytes + val rbsp = ByteBuffer.allocateDirect(rbspSize) - rbsp.put(this, this.startCodeSize, headerLength) + // Write header to new buffer + rbsp.put(this, 0, headerLength + this.startCodeSize) - var previous = this.position() indices.forEach { - rbsp.put(this, previous, it + 2 - previous) - previous = it + 3 // skip emulation_prevention_three_byte + rbsp.put(this, 0, it + 2 - this.position()) + this.position(this.position() + 1) // skip emulation_prevention_three_byte } - rbsp.put(this, previous, this.limit() - previous) + rbsp.put(this, 0, this.limit() - this.position()) - rbsp.limit(rbsp.position()) rbsp.rewind() return rbsp } /** * Remove all [prefixes] from [ByteBuffer] whatever their order. - * It slices [ByteBuffer] so it does not copy data. + * It moves the [position] of the [ByteBuffer]. * * Once a prefix is found, it is removed from the [prefixes] list. * @@ -324,7 +363,7 @@ fun ByteBuffer.removePrefixes(prefixes: List): ByteBuffer { } } - return this.slice().order(this.order()) + return this } /** @@ -343,4 +382,21 @@ val ByteBuffer.isAvcc: Boolean get() { val size = this.getInt(0) return size == (this.remaining() - 4) - } \ No newline at end of file + } + +/** + * Slices the buffer from [from] position to [to] position. + * + * @param from start position + * @param to end position + */ +fun ByteBuffer.slice(from: Int, to: Int): ByteBuffer { + val currentPosition = this.position() + val currentLimit = this.limit() + this.position(from) + this.limit(to) + val newBuffer = this.slice() + this.position(currentPosition) + this.limit(currentLimit) + return newBuffer +} diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 9825800e9..3ceff1035 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -134,12 +134,16 @@ class ByteBufferExtensionsKtTest { val resultBuffers = testBuffer.slices( byteArrayOf(0, 0, 0, 1) ) + assertEquals( 3, resultBuffers.size ) - var resultArray = ByteArray(0) - resultBuffers.forEach { resultArray += it.array() } // concat all arrays - assertArrayEquals(testBuffer.array(), resultArray) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -172,7 +176,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals(testBuffer.array(), resultBuffers[0].array()) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -189,9 +198,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals( - testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffers[0].array() - ) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffer.array()) } @Test @@ -263,6 +275,10 @@ class ByteBufferExtensionsKtTest { ) ) val expectedArray = byteArrayOf( + 0, + 0, + 0, + 1, 66, 1, 1, @@ -336,4 +352,19 @@ class ByteBufferExtensionsKtTest { assertEquals(0, prefixBuffer.position()) assertEquals(0, testBuffer.position()) } + + @Test + fun `slice from to test`() { + val testBuffer = ByteBuffer.wrap("ABCDE".toByteArray()) + val slice = testBuffer.slice(1, 4) + + assertArrayEquals( + "BCDE".toByteArray(), slice.toByteArray() + ) + assertEquals(0, slice.position()) + assertEquals(4, slice.limit()) + + assertEquals(0, testBuffer.position()) + assertEquals(5, testBuffer.limit()) + } } \ No newline at end of file From b2d6c9ffaac8d38ccd58454ae53a03542bbd5a4a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:48:16 +0100 Subject: [PATCH 07/27] feat(core): pool make frame put back themself in the pool after close --- .../thibaultbee/streampack/core/elements/data/Frame.kt | 1 - .../elements/encoders/mediacodec/MediaCodecEncoder.kt | 3 +-- .../streampack/core/elements/utils/pool/FramePool.kt | 9 +++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index e7505acd2..74b154dbc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -108,7 +108,6 @@ fun Frame.copy( return pool.get( rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, { frame -> - pool.put(frame) onClosed(frame) }) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index fe033fb96..9e2d9b97c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -658,10 +658,9 @@ internal constructor( isKeyFrame, extra, outputFormat, - onClosed = { frame -> + onClosed = { try { codec.releaseOutputBuffer(index, false) - pool.put(frame) } catch (t: Throwable) { Logger.w(tag, "Failed to release output buffer for code: ${t.message}") } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 264917486..7302ef272 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -35,6 +35,11 @@ internal class FramePool() : ObjectPool() { ): MutableFrame { val frame = get() + val onClosedHook = { frame: MutableFrame -> + onClosed(frame) + put(frame) + } + return if (frame != null) { frame.rawBuffer = rawBuffer frame.ptsInUs = ptsInUs @@ -42,7 +47,7 @@ internal class FramePool() : ObjectPool() { frame.isKeyFrame = isKeyFrame frame.extra = extra frame.format = format - frame.onClosed = onClosed + frame.onClosed = onClosedHook frame } else { MutableFrame( @@ -52,7 +57,7 @@ internal class FramePool() : ObjectPool() { isKeyFrame = isKeyFrame, extra = extra, format = format, - onClosed = onClosed + onClosed = onClosedHook ) } } From cc083375413b063e16679a19d9bfdc8744a5ff5a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 27 Jan 2026 21:59:38 +0100 Subject: [PATCH 08/27] feat(core): add a raw frame pool to reuse raw frame --- .../core/elements/sources/StubAudioSource.kt | 11 ++-- .../core/elements/sources/StubVideoSource.kt | 7 ++- .../encoding/EncodingPipelineOutputTest.kt | 6 +- .../streampack/core/elements/data/Frame.kt | 48 +++++++++++++-- .../elements/processing/RawFramePullPush.kt | 33 ++++++----- .../audio/IAudioFrameSourceInternal.kt | 20 +++---- .../audio/audiorecord/AudioRecordSource.kt | 35 +++++------ .../sources/video/IVideoFrameSource.kt | 10 ++-- .../elements/utils/pool/IRawFrameFactory.kt | 31 ---------- .../elements/utils/pool/RawFrameFactory.kt | 53 ----------------- .../core/elements/utils/pool/RawFramePool.kt | 58 +++++++++++++++++++ .../core/pipelines/StreamerPipeline.kt | 26 +++------ .../core/pipelines/inputs/AudioInput.kt | 40 ++++++------- .../elements/sources/AudioCaptureUnitTest.kt | 4 +- .../elements/utils/StubRawFrameFactory.kt | 35 ----------- 15 files changed, 184 insertions(+), 233 deletions(-) delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt delete mode 100644 core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt index f929a92b6..4405578e4 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt @@ -1,12 +1,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.content.Context -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubAudioSource : IAudioSourceInternal { private val _isStreamingFlow = MutableStateFlow(false) @@ -15,13 +14,11 @@ class StubAudioSource : IAudioSourceInternal { private val _configurationFlow = MutableStateFlow(null) val configurationFlow = _configurationFlow.asStateFlow() - override fun fillAudioFrame(frame: RawFrame): RawFrame { - return frame + override val minBufferSize = 0 + override fun fillAudioFrame(buffer: ByteBuffer): Long { + return 0L } - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0) - override suspend fun startStream() { _isStreamingFlow.value = true } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt index 22eca54d4..883c6ada1 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt @@ -8,12 +8,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.ISurfaceSour import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.VideoSourceConfig -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.time.Timebase import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME) : StubVideoSource(), @@ -44,8 +44,9 @@ class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME } class StubVideoFrameSource : StubVideoSource(), IVideoFrameSourceInternal { - override fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0L) + override fun getVideoFrame(buffer: ByteBuffer): Long { + return 0L + } class Factory : IVideoSourceInternal.Factory { override suspend fun create( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 917940c47..b2f1d4daf 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -20,7 +20,7 @@ import android.util.Log import androidx.core.net.toFile import androidx.test.platform.app.InstrumentationRegistry import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.DummyEndpoint @@ -317,7 +317,7 @@ class EncodingPipelineOutputTest { try { output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) @@ -330,7 +330,7 @@ class EncodingPipelineOutputTest { output.startStream(descriptor) output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 74b154dbc..c1a952932 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -17,28 +17,64 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.io.Closeable import java.nio.ByteBuffer /** - * A raw frame internal representation. + * Encoded frame representation */ -data class RawFrame( +interface RawFrame : Closeable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - var timestampInUs: Long, + val timestampInUs: Long +} + + +/** + * Copy a [RawFrame] to a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val pool = RawFramePool.default + return pool.get( + rawBuffer, timestampInUs, + { frame -> + onClosed(frame) + }) +} + +/** + * A mutable [RawFrame] internal representation. + * + * The purpose is to get reusable [RawFrame] + */ +data class MutableRawFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + /** + * Presentation timestamp in µs + */ + override var timestampInUs: Long, /** * A callback to call when frame is closed. */ - val onClosed: (RawFrame) -> Unit = {} -) : Closeable { + override var onClosed: (MutableRawFrame) -> Unit = {} +) : RawFrame, WithClosable { override fun close() { try { onClosed(this) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index 3dd308c16..f87b531ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -16,8 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.processing import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFrameFactory +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -35,40 +36,42 @@ fun RawFramePullPush( onFrame: suspend (RawFrame) -> Unit, processDispatcher: CoroutineDispatcher, isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, RawFrameFactory(isDirect), processDispatcher) +) = RawFramePullPush(frameProcessor, onFrame, ByteBufferPool(isDirect), processDispatcher) /** * A component that pull a frame from an input and push it to [onFrame] output. * * @param frameProcessor the frame processor * @param onFrame the output frame callback - * @param frameFactory the frame factory to create frames + * @param bufferPool the [ByteBuffer] pool * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( private val frameProcessor: IFrameProcessor, val onFrame: suspend (RawFrame) -> Unit, - private val frameFactory: IRawFrameFactory, + private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, ) { private val coroutineScope = CoroutineScope(SupervisorJob() + processDispatcher) private val mutex = Mutex() - private var getFrame: (suspend (frameFactory: IRawFrameFactory) -> RawFrame)? = null + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null private val isReleaseRequested = AtomicBoolean(false) private var job: Job? = null - suspend fun setInput(getFrame: suspend (frameFactory: IRawFrameFactory) -> RawFrame) { + suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } @@ -80,9 +83,11 @@ class RawFramePullPush( job = coroutineScope.launch { while (isActive) { val rawFrame = mutex.withLock { - val listener = getFrame ?: return@withLock null + val unwrapSource = source ?: return@withLock null try { - listener(frameFactory) + val buffer = bufferPool.get(unwrapSource.minBufferSize) + val timestampInUs = unwrapSource.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } catch (t: Throwable) { Logger.e(TAG, "Failed to get frame: ${t.message}") null @@ -115,7 +120,8 @@ class RawFramePullPush( job?.cancel() job = null - frameFactory.clear() + pool.clear() + bufferPool.clear() } fun release() { @@ -127,7 +133,8 @@ class RawFramePullPush( } coroutineScope.cancel() - frameFactory.close() + pool.close() + bufferPool.close() } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt index 18c89bb6c..209ae9ea8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt @@ -15,26 +15,20 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.audio -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IAudioFrameSourceInternal { /** - * Gets an audio frame from the source. - * - * @param frame the [RawFrame] to fill with audio data. - * @return a [RawFrame] containing audio data. + * Gets the size of the buffer to allocate. + * When using encoder is callback mode, it's unused. */ - fun fillAudioFrame(frame: RawFrame): RawFrame + val minBufferSize: Int /** * Gets an audio frame from the source. * - * The [RawFrame] to fill with audio data is created by the [frameFactory]. - * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing audio data. + * @param buffer the [ByteBuffer] to fill with audio data. + * @return the timestamp in microseconds. */ - fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun fillAudioFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt index 324438aaf..cc1746614 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt @@ -22,18 +22,18 @@ import android.media.AudioTimestamp import android.media.audiofx.AudioEffect import android.os.Build import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Companion.isValidUUID import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.getFactoryForEffectType +import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.availableEffect import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.isEffectAvailable -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.extensions.type -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer import java.util.UUID /** @@ -42,7 +42,9 @@ import java.util.UUID */ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSource { private var audioRecord: AudioRecord? = null - private var bufferSize: Int? = null + private var _minBufferSize: Int? = null + override val minBufferSize: Int + get() = _minBufferSize ?: throw IllegalStateException("Audio source is not initialized") private var processor: EffectProcessor? = null private var pendingAudioEffects = mutableListOf() @@ -74,9 +76,9 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } } - bufferSize = getMinBufferSize(config) + _minBufferSize = getMinBufferSize(config) - audioRecord = buildAudioRecord(config, bufferSize!!).also { + audioRecord = buildAudioRecord(config, _minBufferSize!!).also { val previousEffects = processor?.getAll() ?: emptyList() processor?.clear() @@ -157,30 +159,19 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } - override fun fillAudioFrame(frame: RawFrame): RawFrame { + override fun fillAudioFrame( + buffer: ByteBuffer + ): Long { val audioRecord = requireNotNull(audioRecord) { "Audio source is not initialized" } if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { throw IllegalStateException("Audio source is not recording") } - val buffer = frame.rawBuffer val length = audioRecord.read(buffer, buffer.remaining()) - if (length > 0) { - frame.timestampInUs = getTimestampInUs(audioRecord) - return frame - } else { - frame.close() + if (length <= 0) { throw IllegalArgumentException(audioRecordErrorToString(length)) } - } - - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame { - val bufferSize = requireNotNull(bufferSize) { "Buffer size is not initialized" } - - /** - * Dummy timestamp: it is overwritten later. - */ - return fillAudioFrame(frameFactory.create(bufferSize, 0)) + return getTimestampInUs(audioRecord) } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt index 3c206900d..a90d61103 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt @@ -15,17 +15,15 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IVideoFrameSourceInternal { /** * Gets a video frame from a source. * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing video data. + * @param buffer The buffer to fill with the video frame. + * @return The timestamp in microseconds of the video frame. */ - fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun getVideoFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt deleted file mode 100644 index 759419483..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IRawFrameFactory.kt +++ /dev/null @@ -1,31 +0,0 @@ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -interface IReadOnlyRawFrameFactory { - /** - * Creates a [RawFrame]. - * - * The returned frame must be released by calling [RawFrame.close] when it is not used anymore. - * - * @param bufferSize the buffer size - * @param timestampInUs the frame timestamp in µs - * @return a frame - */ - fun create(bufferSize: Int, timestampInUs: Long): RawFrame -} - -/** - * A pool of frames. - */ -interface IRawFrameFactory : IReadOnlyRawFrameFactory { - /** - * Clears the factory. - */ - fun clear() - - /** - * Closes the factory. - */ - fun close() -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt deleted file mode 100644 index dc4b3032b..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.logger.Logger - -/** - * A factory to create [RawFrame]. - */ -fun RawFrameFactory(isDirect: Boolean): RawFrameFactory { - return RawFrameFactory(ByteBufferPool(isDirect)) -} - -/** - * A factory to create [RawFrame]. - */ -class RawFrameFactory(private val bufferPool: ByteBufferPool) : IRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - try { - bufferPool.put(rawFrame.rawBuffer) - } catch (t: Throwable) { - Logger.w(TAG, "Error while putting buffer in pool: $t") - } - } - } - - override fun clear() { - bufferPool.clear() - } - - override fun close() { - bufferPool.close() - } - - companion object { - private const val TAG = "RawFramePool" - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt new file mode 100644 index 000000000..0c7b4e6a0 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame +import java.nio.ByteBuffer + + +/** + * A pool of [MutableRawFrame]. + */ +internal class RawFramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + timestampInUs: Long, + onClosed: (MutableRawFrame) -> Unit = {} + ): MutableRawFrame { + val frame = get() + + val onClosedHook = { frame: MutableRawFrame -> + onClosed(frame) + put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.timestampInUs = timestampInUs + frame.onClosed = onClosedHook + frame + } else { + MutableRawFrame( + rawBuffer = rawBuffer, + timestampInUs = timestampInUs, + onClosed = onClosedHook + ) + } + } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { RawFramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index bb745019c..26d417892 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.pipelines import android.content.Context import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory @@ -70,7 +71,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.Closeable import java.util.concurrent.atomic.AtomicBoolean /** @@ -246,25 +246,17 @@ open class StreamerPipeline( Logger.e(TAG, "Error while queueing audio frame to output: $t") } } else { - // Hook to close frame when all outputs have processed it - var numOfClosed = 0 - val onClosed = { frame: Closeable -> - numOfClosed++ - if (numOfClosed == audioStreamingOutput.size) { - frame.close() - } - } audioStreamingOutput.forEachIndexed { index, output -> try { output.queueAudioFrame( - frame.copy( - rawBuffer = if (index == audioStreamingOutput.lastIndex) { - frame.rawBuffer - } else { - frame.rawBuffer.duplicate() - }, - onClosed = onClosed - ) + if (index == audioStreamingOutput.lastIndex) { + frame + } else { + frame.copy( + rawBuffer = + frame.rawBuffer.duplicate() + ) + } ) } catch (t: Throwable) { Logger.e(TAG, "Error while queueing audio frame to output $output: $t") diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 05812e4d8..9560378e1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -24,10 +24,11 @@ import io.github.thibaultbee.streampack.core.elements.processing.RawFramePullPus import io.github.thibaultbee.streampack.core.elements.processing.audio.AudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.processing.audio.IAudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING import io.github.thibaultbee.streampack.core.pipelines.IAudioDispatcherProvider @@ -200,15 +201,7 @@ internal class AudioInput( } } - when (port) { - is PushAudioPort -> { - port.setInput(newAudioSource::getAudioFrame) - } - - is CallbackAudioPort -> { - port.setInput(newAudioSource::fillAudioFrame) - } - } + port.setInput(newAudioSource) // Replace audio source sourceInternalFlow.emit(newAudioSource) @@ -380,8 +373,8 @@ internal class AudioInput( internal class CallbackConfig : Config() } -private sealed interface IAudioPort : Streamable, Releasable { - suspend fun setInput(getFrame: T) +private sealed interface IAudioPort : Streamable, Releasable { + suspend fun setInput(source: IAudioFrameSourceInternal) suspend fun removeInput() } @@ -389,7 +382,7 @@ private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, dispatcherProvider: IAudioDispatcherProvider -) : IAudioPort<(frameFactory: IRawFrameFactory) -> RawFrame> { +) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, @@ -399,8 +392,8 @@ private class PushAudioPort( ) ) - override suspend fun setInput(getFrame: (frameFactory: IRawFrameFactory) -> RawFrame) { - audioPullPush.setInput(getFrame) + override suspend fun setInput(source: IAudioFrameSourceInternal) { + audioPullPush.setInput(source) } override suspend fun removeInput() { @@ -421,32 +414,35 @@ private class PushAudioPort( } private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProcessor) : - IAudioPort<(frame: RawFrame) -> RawFrame> { - private var getFrame: ((frame: RawFrame) -> RawFrame)? = null + IAudioPort { private val mutex = Mutex() + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null var audioFrameRequestedListener: OnFrameRequestedListener = object : OnFrameRequestedListener { override suspend fun onFrameRequested(buffer: ByteBuffer): RawFrame { val frame = mutex.withLock { - val getFrame = requireNotNull(getFrame) { + val source = requireNotNull(source) { "Audio frame requested listener is not set yet" } - getFrame(RawFrame(buffer, 0)) + val timestampInUs = source.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } return audioFrameProcessor.processFrame(frame) } } - override suspend fun setInput(getFrame: (frame: RawFrame) -> RawFrame) { + override suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } override suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt index 2c579cdcd..393c6a473 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt @@ -18,11 +18,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.media.MediaRecorder import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSource import io.github.thibaultbee.streampack.core.elements.utils.StubLogger -import io.github.thibaultbee.streampack.core.elements.utils.StubRawFrameFactory import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.test.runTest import org.junit.Assert import org.junit.Test +import java.nio.ByteBuffer class MicrophoneSourceUnitTest { init { @@ -38,7 +38,7 @@ class MicrophoneSourceUnitTest { } catch (_: Throwable) { } try { - microphoneSource.getAudioFrame(StubRawFrameFactory()) + microphoneSource.fillAudioFrame(ByteBuffer.allocate(microphoneSource.minBufferSize)) Assert.fail() } catch (_: Throwable) { } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt deleted file mode 100644 index a4d24568a..000000000 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IGetOnlyBufferPool -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory -import java.nio.ByteBuffer - -/** - * Stub buffer pool for testing. - * - * It always returns a new allocated buffer. - */ -class StubRawFrameFactory(private val bufferPool: IGetOnlyBufferPool = StubBufferPool()) : - IReadOnlyRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - // Do nothing - } - } -} \ No newline at end of file From 0e18b1cf5c8bddaea361efc3289c1f8289e24894 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:14:05 +0100 Subject: [PATCH 09/27] feat(*): avoid to duplicate extra for each frames --- .../streampack/core/elements/data/Frame.kt | 51 +++++++++++++++++-- .../encoders/mediacodec/MediaCodecEncoder.kt | 42 ++++++++------- .../composites/muxers/mp4/models/Chunk.kt | 4 +- .../endpoints/composites/muxers/ts/TsMuxer.kt | 24 +++++---- .../core/elements/utils/pool/FramePool.kt | 3 +- .../core/elements/utils/FakeFrames.kt | 9 ++-- .../muxer/utils/FlvAudioDataFactory.kt | 7 +-- .../muxer/utils/FlvVideoDataFactory.kt | 16 +++--- 8 files changed, 110 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index c1a952932..c9c08acea 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -109,11 +109,12 @@ interface Frame : Closeable { val isKeyFrame: Boolean /** - * Contains csd buffers for key frames and audio frames only. + * Gets the csd buffers for key frames and audio frames. + * * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List? + val extra: Extra? /** * Contains frame format.. @@ -136,7 +137,7 @@ fun Frame.copy( ptsInUs: Long = this.ptsInUs, dtsInUs: Long? = this.dtsInUs, isKeyFrame: Boolean = this.isKeyFrame, - extra: List? = this.extra, + extra: Extra? = this.extra, format: MediaFormat = this.format, onClosed: (Frame) -> Unit = {} ): Frame { @@ -180,7 +181,7 @@ data class MutableFrame( * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - override var extra: List?, + override var extra: Extra?, /** * Contains frame format.. @@ -201,3 +202,45 @@ data class MutableFrame( } } } + +/** + * Ensures that extra are not used at the same time. + * + * After accessing the extra, they are automatically rewind for a new usage. + */ +class Extra(private val extraBuffers: List) { + private val lock = Any() + + val _length by lazy { extraBuffers.sumOf { it.remaining() } } + + fun getLength(): Int { + return synchronized(lock) { + _length + } + } + + fun get(extra: List.() -> T): T { + return synchronized(lock) { + val result = extraBuffers.extra() + extraBuffers.forEach { it.rewind() } + result + } + } +} + +/** + * Gets the duplicated extra. + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +val Extra.extra: List + get() = get { this.map { it.duplicate() } } + +/** + * Gets the duplicated extra at [index] + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +fun Extra.get(index: Int): ByteBuffer { + return get { this[index].duplicate() } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index 9e2d9b97c..9ebbde6a6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,6 +22,7 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode @@ -63,14 +64,8 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat - private var outputFormat: MediaFormat? = null - set(value) { - extra = value?.extra - field = value - } - private var extra: List? = null - private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo, mediaCodec.outputFormat) } private val isVideo = encoderConfig.isVideo private val tag = if (isVideo) VIDEO_ENCODER_TAG else AUDIO_ENCODER_TAG + "(${this.hashCode()})" @@ -360,7 +355,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, extra, outputFormat!!, info, tag + index, info, tag ) try { listener.outputChannel.send(frame) @@ -444,7 +439,6 @@ internal constructor( } override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - outputFormat = format Logger.i(tag, "Format changed : $format") } @@ -566,7 +560,6 @@ internal constructor( if (outputBufferId >= 0) { processOutputFrameUnsafe(mediaCodec, outputBufferId, bufferInfo) } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - outputFormat = mediaCodec.outputFormat Logger.i(tag, "Format changed: ${mediaCodec.outputFormat}") } } @@ -587,6 +580,18 @@ internal constructor( } } + fun FrameFactory( + codec: MediaCodec, + isVideo: Boolean, + outputFormat: MediaFormat + ): FrameFactory { + val extraBuffers: List? by lazy { outputFormat.extra } + val extra: Extra? by lazy { + extraBuffers?.map { it.duplicate() }?.let { Extra(it) } + } + return FrameFactory(codec, isVideo, outputFormat, extra, extraBuffers) + } + /** * A workaround to address the fact that some AAC encoders do not provide frame with `presentationTimeUs` in order. * If a frame is received with a timestamp lower or equal to the previous one, it is corrected by adding 1 to the previous timestamp. @@ -596,7 +601,10 @@ internal constructor( */ class FrameFactory( private val codec: MediaCodec, - private val isVideo: Boolean + private val isVideo: Boolean, + private val outputFormat: MediaFormat, + private val extra: Extra?, + private val extraBuffers: List? ) : Closeable { private var previousPresentationTimeUs = 0L @@ -609,8 +617,6 @@ internal constructor( */ fun frame( index: Int, - extra: List?, - outputFormat: MediaFormat, info: BufferInfo, tag: String ): Frame { @@ -620,7 +626,7 @@ internal constructor( Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, extra, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, pts, info.isKeyFrame, tag) } /** @@ -633,20 +639,18 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, - extra: List?, - outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, tag: String ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) val extra = if (isKeyFrame || !isVideo) { - extra!!.map { it.duplicate() } + extra!! } else { null } - val rawBuffer = if (extra != null) { - buffer.removePrefixes(extra) + val rawBuffer = if (extraBuffers != null) { + buffer.removePrefixes(extraBuffers) } else { buffer } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt index 2ff9387c4..373b6eb42 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.unzip import java.nio.ByteBuffer @@ -50,8 +51,9 @@ class Chunk(val id: Int) { private val sampleSizes: List get() = samples.map { it.frame.rawBuffer.remaining() } + // TODO: pass `Extra` instead of `List` val extra: List> - get() = samples.mapNotNull { it.frame.extra }.unzip() + get() = samples.mapNotNull { it.frame.extra?.extra }.unzip() val format: List get() = samples.map { it.frame.format } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index ba17e1291..127662e43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -88,12 +88,14 @@ class TsMuxer : IMuxerInternal { val extra = frame.extra ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -108,13 +110,15 @@ class TsMuxer : IMuxerInternal { val extra = frame.extra ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -130,11 +134,13 @@ class TsMuxer : IMuxerInternal { frame.rawBuffer, pes.stream.config as AudioCodecConfig ).toByteBuffer() } else { - LATMFrameWriter.fromDecoderSpecificInfo( - frame.rawBuffer, - frame.extra!!.first() - ) - .toByteBuffer() + frame.extra!!.get { + LATMFrameWriter.fromDecoderSpecificInfo( + frame.rawBuffer, + this.first() + ) + .toByteBuffer() + } } ) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt index 7302ef272..c1e824e97 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.utils.pool import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer @@ -29,7 +30,7 @@ internal class FramePool() : ObjectPool() { ptsInUs: Long, dtsInUs: Long?, isKeyFrame: Boolean, - extra: List?, + extra: Extra?, format: MediaFormat, onClosed: (MutableFrame) -> Unit ): MutableFrame { diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index b31ab2b44..542ef0a8e 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer @@ -56,9 +57,11 @@ object FakeFrames { pts, dts, isKeyFrame, - listOf( - ByteBuffer.wrap( - Random.nextBytes(10) + Extra( + listOf( + ByteBuffer.wrap( + Random.nextBytes(10) + ) ) ), format = format diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt index b0d1013b5..54df39c08 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt @@ -24,6 +24,7 @@ import io.github.komedia.komuxer.flv.tags.audio.ExtendedAudioDataFactory import io.github.komedia.komuxer.flv.tags.audio.codedFrame import io.github.komedia.komuxer.flv.tags.audio.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.AudioFlvMuxerInfo @@ -72,7 +73,7 @@ internal class FlvAudioDataFactory { val flvDatas = mutableListOf() if (withSequenceStart) { - val decoderConfigurationRecordBuffer = frame.extra!![0] + val decoderConfigurationRecordBuffer = frame.extra!!.get(0) flvDatas.add( aacAudioDataFactory.sequenceStart( decoderConfigurationRecordBuffer @@ -155,7 +156,7 @@ internal class FlvAudioDataFactory { return FlvExtendedAudioDataFactory( ExtendedAudioDataFactory(AudioFourCC.AAC), onSequenceStart = { frame -> - frame.extra!![0] + frame.extra!!.get(0) } ) } @@ -165,7 +166,7 @@ internal class FlvAudioDataFactory { ExtendedAudioDataFactory(AudioFourCC.OPUS), onSequenceStart = { frame -> frame.extra?.let { - OpusCsdParser.findIdentificationHeader(it[0]) + OpusCsdParser.findIdentificationHeader(it.get(0)) } } ) diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt index 6b541bc92..dccc05e0d 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt @@ -28,6 +28,8 @@ import io.github.komedia.komuxer.flv.tags.video.VideoFrameType import io.github.komedia.komuxer.flv.tags.video.codedFrame import io.github.komedia.komuxer.flv.tags.video.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord @@ -64,10 +66,11 @@ internal class FlvVideoDataFactory { VideoFrameType.INTER } if (frame.isKeyFrame && withSequenceStart) { + val extra = frame.extra!!.extra val decoderConfigurationRecordBuffer = AVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1] + extra[0], + extra[1] ).toByteBuffer() flvDatas.add( factory.sequenceStart( @@ -185,10 +188,11 @@ internal class FlvVideoDataFactory { private fun createHEVCFactory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(HEVCExtendedVideoDataFactory()) { frame -> // Extra is VPS, SPS, PPS + val extra = frame.extra!!.extra HEVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1], - frame.extra!![2] + extra[0], + extra[1], + extra[2] ).toByteBuffer() } } @@ -196,7 +200,7 @@ internal class FlvVideoDataFactory { private fun createAV1Factory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(ExtendedVideoDataFactory(VideoFourCC.AV1)) { frame -> // Extra is AV1CodecConfigurationRecord - frame.extra!![0] + frame.extra!!.get(0) } } From c15e48b23693b2fcc1ada129baa55487d8e307ce Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:08:26 +0100 Subject: [PATCH 10/27] refactor(core): camera: move `applyRepeatingSessionSync` method to the camera settings --- .../sources/video/camera/CameraSettings.kt | 60 +++++++++- .../camera/controllers/CameraController.kt | 15 ++- .../controllers/CameraSessionController.kt | 107 ++++-------------- 3 files changed, 89 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index bc87cc182..c532ff208 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -22,6 +22,7 @@ import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.hardware.camera2.params.ColorSpaceTransform import android.hardware.camera2.params.MeteringRectangle import android.hardware.camera2.params.RggbChannelVector @@ -32,6 +33,7 @@ import androidx.annotation.IntRange import androidx.annotation.RequiresApi import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoFocusModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoWhiteBalanceModes @@ -55,10 +57,12 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.launchIn import io.github.thibaultbee.streampack.core.elements.utils.extensions.normalize import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicLong /** @@ -129,6 +133,8 @@ class CameraSettings internal constructor( val focusMetering = FocusMetering(coroutineScope, characteristics, this, zoom, focus, exposure, whiteBalance) + private val tagBundleFactory = TagBundle.TagBundleFactory() + /** * Directly gets a [CaptureRequest] from the camera. * @@ -153,13 +159,65 @@ class CameraSettings internal constructor( * * This method returns when the capture callback is received with the passed request. */ - suspend fun applyRepeatingSessionSync() = cameraController.setRepeatingSessionSync() + suspend fun applyRepeatingSessionSync() { + val deferred = CompletableDeferred() + + val tag = tagBundleFactory.create() + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + val resultTag = result.request.tag as? TagBundle + val keyId = resultTag?.keyId ?: return false + if (keyId >= tag.keyId) { + deferred.complete(Unit) + return true + } + return false + } + } + + cameraController.addCaptureCallbackListener(captureCallback) + cameraController.setRepeatingSession(tag) + deferred.await() + } /** * Applies settings to the camera repeatedly. */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() + private class TagBundle private constructor(val keyId: Long) { + private val tagMap = mutableMapOf().apply { + put(TAG_KEY_ID, keyId) + } + + val keys: Set + get() = tagMap.keys + + fun setTag(key: String, value: Any?) { + tagMap[key] = value + } + + companion object { + private const val TAG_KEY_ID = "TAG_KEY_ID" + } + + /** + * Factory for [TagBundle]. + * + * The purpose is to make sure the tag always contains an increasing id. + */ + class TagBundleFactory { + /** + * Next session id. + */ + private val nextSessionUpdateId = AtomicLong(0) + + fun create(): TagBundle { + return TagBundle(nextSessionUpdateId.getAndIncrement()) + } + } + } + class Flash( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 13c33e653..82713b589 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -21,6 +21,7 @@ import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.util.Range import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface @@ -378,21 +379,23 @@ internal class CameraController( } /** - * Sets a repeating session sync with the current capture request. + * Adds a capture callback listener to the current capture session. * - * It returns only when the capture callback has been called for the first time. + * The listener is removed when the [CaptureResultListener] returns true. */ - suspend fun setRepeatingSessionSync() { + fun addCaptureCallbackListener(listener: CaptureResultListener) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSessionSync() + sessionController.addCaptureCallbackListener(listener) } /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { + suspend fun setRepeatingSession(tag: Any? = null) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSession() + sessionController.setRepeatingSession(tag) } private suspend fun closeControllers() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index fc3843e3f..80d768435 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -37,9 +37,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.atomic.AtomicLong -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, @@ -58,8 +55,6 @@ internal class CameraSessionController private constructor( private val requestTargetMutex = Mutex() - private val nextSessionUpdateId = AtomicLong(0) - /** * A default capture callback that logs the failure reason. */ @@ -75,7 +70,7 @@ internal class CameraSessionController private constructor( private val sessionCallback = CameraControlSessionCallback(coroutineScope) private val captureCallbacks = - mutableSetOf(captureCallback, sessionCallback) + setOf(captureCallback, sessionCallback) suspend fun isEmpty() = withContext(coroutineScope.coroutineContext) { requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } @@ -250,72 +245,21 @@ internal class CameraSessionController private constructor( } } - suspend fun removeCaptureCallback( - cameraCaptureCallback: CaptureCallback - ) { - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - captureCallbacks.remove(cameraCaptureCallback) - } - } - } - /** - * Sets a repeating session with the current capture request. + * Adds a capture callback listener to the current capture session. + * + * The listener is removed when the session is closed. */ - suspend fun setRepeatingSessionSync() { - if (captureRequestBuilder.isEmpty()) { - Logger.w(TAG, "Capture request is empty") - return - } - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - suspendCoroutine { continuation -> - val nextSessionId = nextSessionUpdateId.getAndIncrement() - - val captureCallback = object : CaptureResultListener { - override fun onCaptureResult(result: TotalCaptureResult): Boolean { - val tag = result.request.tag as? TagBundle - val keyId = tag?.keyId ?: return false - if (keyId >= nextSessionId) { - continuation.resume(Unit) - return true - } - return false - } - } - - sessionCallback.addListener(captureCallback) - - captureRequestBuilder.setTag( - TagBundle().apply { - keyId = nextSessionId - } - ) - sessionCompat.setRepeatingSingleRequest( - captureSession, - captureRequestBuilder.build(), - MultiCaptureCallback(captureCallbacks) - ) - } - } - } + fun addCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.addListener(listener) } /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { + suspend fun setRepeatingSession(tag: Any? = null) { if (captureRequestBuilder.isEmpty()) { Logger.w(TAG, "Capture request is empty") return @@ -327,6 +271,8 @@ internal class CameraSessionController private constructor( return@withContext } + tag?.let { captureRequestBuilder.setTag(it) } + sessionCompat.setRepeatingSingleRequest( captureSession, captureRequestBuilder.build(), @@ -514,27 +460,6 @@ internal class CameraSessionController private constructor( } } - private class TagBundle { - private val tagMap = mutableMapOf() - - var keyId: Long? - get() = tagMap[TAG_KEY_ID] as? Long - set(value) { - tagMap[TAG_KEY_ID] = value - } - - val keys: Set - get() = tagMap.keys - - fun setTag(key: String, value: Any?) { - tagMap[key] = value - } - - companion object { - private const val TAG_KEY_ID = "TAG_KEY_ID" - } - } - interface CaptureResultListener { /** * Called when a capture result is received. @@ -545,11 +470,21 @@ internal class CameraSessionController private constructor( fun onCaptureResult(result: TotalCaptureResult): Boolean } + /** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : CaptureCallback() { /* synthetic accessor */ private val resultListeners = mutableSetOf() + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ fun addListener(listener: CaptureResultListener) { resultListeners.add(listener) } From 9d03ffeb1c5ff01a421bbbbd1701856a70d854c4 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:21:46 +0100 Subject: [PATCH 11/27] feat(core): add a flow to get the current physical camera Id --- .../sources/video/camera/CameraSettings.kt | 26 +++++++++++++++++++ .../camera/controllers/CameraController.kt | 15 ++++++++++- .../controllers/CameraSessionController.kt | 22 +++++++++++----- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c532ff208..9b0a41b1a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -60,6 +60,11 @@ import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicLong @@ -81,6 +86,27 @@ class CameraSettings internal constructor( */ val cameraId = cameraController.cameraId + @RequiresApi(Build.VERSION_CODES.Q) + private fun getPhysicalCameraIdCallbackFlow() = callbackFlow { + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + trySend(result.get(CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID)!!) + return false + } + } + cameraController.addCaptureCallbackListener(captureCallback) + awaitClose { + cameraController.removeCaptureCallbackListener(captureCallback) + } + }.conflate().distinctUntilChanged() + + /** + * Current physical camera id. + */ + val physicalCameraIdFlow: Flow + @RequiresApi(Build.VERSION_CODES.Q) + get() = getPhysicalCameraIdCallbackFlow() + /** * Whether the camera is available. * To be used before calling any camera settings. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 82713b589..5f7ffce5c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -381,13 +381,26 @@ internal class CameraController( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the [CaptureResultListener] returns true. + * The listener is removed when the [CaptureResultListener] returns true or [removeCaptureCallbackListener] is called. + * + * @param listener The listener to add */ fun addCaptureCallbackListener(listener: CaptureResultListener) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } sessionController.addCaptureCallbackListener(listener) } + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + fun removeCaptureCallbackListener(listener: CaptureResultListener) { + val sessionController = requireNotNull(sessionController) { "SessionController is null" } + sessionController.removeCaptureCallbackListener(listener) + } + + /** * Sets a repeating session with the current capture request. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index 80d768435..ab014f2eb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -40,9 +40,10 @@ import kotlinx.coroutines.withContext internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, + private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, + private val sessionCallback: CameraControlSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, - private val captureSession: CameraCaptureSession, private val outputs: List, val dynamicRange: Long, val cameraIsClosedFlow: StateFlow, @@ -67,8 +68,6 @@ internal class CameraSessionController private constructor( } } - private val sessionCallback = CameraControlSessionCallback(coroutineScope) - private val captureCallbacks = setOf(captureCallback, sessionCallback) @@ -248,12 +247,21 @@ internal class CameraSessionController private constructor( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the session is closed. + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ fun addCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.addListener(listener) } + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + fun removeCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.removeListener(listener) + } + /** * Sets a repeating session with the current capture request. * @@ -360,9 +368,10 @@ internal class CameraSessionController private constructor( val controller = CameraSessionController( coroutineScope, + newCaptureSession, captureRequestBuilder, + sessionCallback, sessionCompat, - newCaptureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, @@ -411,9 +420,10 @@ internal class CameraSessionController private constructor( } return CameraSessionController( coroutineScope, + captureSession, captureRequestBuilder, + CameraControlSessionCallback(coroutineScope), sessionCompat, - captureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, From 57da895e9c94edac765b8852716daed456e4f698 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:52:06 +0100 Subject: [PATCH 12/27] refactor(core): camera: move session callback out of session controller --- .../camera/controllers/CameraController.kt | 17 +++-- .../controllers/CameraSessionController.kt | 69 ++----------------- .../camera/utils/CameraSessionCallback.kt | 67 ++++++++++++++++++ 3 files changed, 80 insertions(+), 73 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 5f7ffce5c..ec7000895 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -24,6 +24,7 @@ import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder @@ -58,6 +59,8 @@ internal class CameraController( private var deviceController: CameraDeviceController? = null private var sessionController: CameraSessionController? = null + private val sessionCallback = CameraSessionCallback(coroutineScope) + private val controllerMutex = Mutex() private val outputs = mutableMapOf() @@ -161,8 +164,9 @@ internal class CameraController( val deviceController = getDeviceController() CameraSessionController.create( coroutineScope, - sessionCompat, deviceController, + sessionCallback, + sessionCompat, outputs.values.toList(), dynamicRange = dynamicRangeProfile.dynamicRange, fpsRange = fpsRange, @@ -381,13 +385,10 @@ internal class CameraController( /** * Adds a capture callback listener to the current capture session. * - * The listener is removed when the [CaptureResultListener] returns true or [removeCaptureCallbackListener] is called. - * - * @param listener The listener to add + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ fun addCaptureCallbackListener(listener: CaptureResultListener) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.addCaptureCallbackListener(listener) + sessionCallback.addListener(listener) } /** @@ -396,11 +397,9 @@ internal class CameraController( * @param listener The listener to remove */ fun removeCaptureCallbackListener(listener: CaptureResultListener) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.removeCaptureCallbackListener(listener) + sessionCallback.removeListener(listener) } - /** * Sets a repeating session with the current capture request. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index ab014f2eb..ab193b4ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -23,6 +23,7 @@ import android.hardware.camera2.TotalCaptureResult import android.util.Range import android.view.Surface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder @@ -42,7 +43,7 @@ internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, - private val sessionCallback: CameraControlSessionCallback, + private val sessionCallback: CameraSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, private val outputs: List, val dynamicRange: Long, @@ -244,24 +245,6 @@ internal class CameraSessionController private constructor( } } - /** - * Adds a capture callback listener to the current capture session. - * - * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. - */ - fun addCaptureCallbackListener(listener: CaptureResultListener) { - sessionCallback.addListener(listener) - } - - /** - * Removes a capture callback listener from the current capture session. - * - * @param listener The listener to remove - */ - fun removeCaptureCallbackListener(listener: CaptureResultListener) { - sessionCallback.removeListener(listener) - } - /** * Sets a repeating session with the current capture request. * @@ -391,8 +374,9 @@ internal class CameraSessionController private constructor( suspend fun create( coroutineScope: CoroutineScope, - sessionCompat: ICameraCaptureSessionCompat, cameraDeviceController: CameraDeviceController, + sessionCallback: CameraSessionCallback, + sessionCompat: ICameraCaptureSessionCompat, outputs: List, dynamicRange: Long, fpsRange: Range, @@ -422,7 +406,7 @@ internal class CameraSessionController private constructor( coroutineScope, captureSession, captureRequestBuilder, - CameraControlSessionCallback(coroutineScope), + sessionCallback, sessionCompat, outputs, dynamicRange, @@ -479,47 +463,4 @@ internal class CameraSessionController private constructor( */ fun onCaptureResult(result: TotalCaptureResult): Boolean } - - /** - * A capture callback that wraps multiple [CaptureResultListener]. - * - * @param coroutineScope The coroutine scope to use. - */ - private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : - CaptureCallback() { - /* synthetic accessor */ - private val resultListeners = mutableSetOf() - - /** - * Adds a capture result listener. - * - * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. - */ - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) - } - - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) - } - - override fun onCaptureCompleted( - session: CameraCaptureSession, - request: CaptureRequest, - result: TotalCaptureResult - ) { - coroutineScope.launch { - val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) - } - } - if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) - } - } - } - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt new file mode 100644 index 000000000..ae8a8ad2c --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCaptureSession.CaptureCallback +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.TotalCaptureResult +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch + +/** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ +internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) : + CaptureCallback() { + /* synthetic accessor */ + private val resultListeners = mutableSetOf() + + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ + fun addListener(listener: CaptureResultListener) { + resultListeners.add(listener) + } + + fun removeListener(listener: CaptureResultListener) { + resultListeners.remove(listener) + } + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + coroutineScope.launch { + val removeSet = mutableSetOf() + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } + } + if (!removeSet.isEmpty()) { + resultListeners.removeAll(removeSet) + } + } + } +} \ No newline at end of file From 404c32f8cf78054559d9d7c2abe9f2e485003267 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:51:40 +0100 Subject: [PATCH 13/27] feat(core): add custom audio effect API in the audio processor --- .../elements/processing/IEffectProcessor.kt | 62 ++++++++++++++++ .../elements/processing/IFrameProcessor.kt | 28 -------- .../elements/processing/RawFramePullPush.kt | 6 +- .../elements/processing/audio/AudioEffects.kt | 37 +++++++--- .../processing/audio/AudioFrameProcessor.kt | 72 ++++++++++++++++--- .../processing/audio/IAudioFrameProcessor.kt | 2 +- .../core/pipelines/inputs/AudioInput.kt | 19 +++-- 7 files changed, 170 insertions(+), 56 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt delete mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt new file mode 100644 index 000000000..0bad99623 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2025 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.processing + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IProcessor { + /** + * Process a data and returns a result. + * + * @param data data to process + * @return processed data + */ + fun process(data: T): T +} + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IEffectProcessor { + /** + * Process a data and returns a result. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + * @return processed data + */ + fun process(isMuted: Boolean, data: T): T +} + +/** + * Interface to process a data. + * + * @param T type of data to process + */ +interface IEffectConsumer { + /** + * Process a data. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + */ + fun consume(isMuted: Boolean, data: T) +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt deleted file mode 100644 index dc9375392..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.processing - -import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -/** - * Interface to process a frame. - * - * @param T type of frame to process (probably [RawFrame] or [Frame]) - */ -interface IFrameProcessor { - fun processFrame(frame: T): T -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index f87b531ad..eca215f31 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -32,7 +32,7 @@ import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean fun RawFramePullPush( - frameProcessor: IFrameProcessor, + frameProcessor: IProcessor, onFrame: suspend (RawFrame) -> Unit, processDispatcher: CoroutineDispatcher, isDirect: Boolean = true @@ -47,7 +47,7 @@ fun RawFramePullPush( * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( - private val frameProcessor: IFrameProcessor, + private val frameProcessor: IProcessor, val onFrame: suspend (RawFrame) -> Unit, private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, @@ -99,7 +99,7 @@ class RawFramePullPush( // Process buffer with effects val processedFrame = try { - frameProcessor.processFrame(rawFrame) + frameProcessor.process(rawFrame) } catch (t: Throwable) { Logger.e(TAG, "Failed to pre-process frame: ${t.message}") continue diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt index 83af9a5a7..bde293f80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt @@ -16,30 +16,47 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IEffectConsumer +import io.github.thibaultbee.streampack.core.elements.processing.IEffectProcessor import java.io.Closeable /** * The base audio effect. */ -interface IAudioEffect : IFrameProcessor, Closeable +sealed interface IAudioEffect : Closeable /** - * Mute audio effect. + * An audio effect that can be dispatched to another thread. The result is not use by the audio pipeline. + * Example: a VU meter. */ -class MuteEffect : IAudioEffect { +interface IConsumerAudioEffect : IAudioEffect, IEffectConsumer + +/** + * An audio effect that can't be dispatched to another thread. The result is used by the audio pipeline. + * + * The [RawFrame.rawBuffer] can't be modified. + */ +interface IProcessorAudioEffect : IAudioEffect, IEffectProcessor + +/** + * An audio effect that mute the audio. + */ +class MuteEffect : IProcessorAudioEffect { private var mutedByteArray: ByteArray? = null - override fun processFrame(frame: RawFrame): RawFrame { - val remaining = frame.rawBuffer.remaining() - val position = frame.rawBuffer.position() + override fun process(isMuted: Boolean, data: RawFrame): RawFrame { + if (!isMuted) { + return data + } + val remaining = data.rawBuffer.remaining() + val position = data.rawBuffer.position() if (remaining != mutedByteArray?.size) { mutedByteArray = ByteArray(remaining) } - frame.rawBuffer.put(mutedByteArray!!) - frame.rawBuffer.position(position) + data.rawBuffer.put(mutedByteArray!!) + data.rawBuffer.position(position) - return frame + return data } override fun close() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 4cd6e7b7a..99c586a22 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,22 +16,76 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import java.io.Closeable +import java.util.function.IntFunction /** * Audio frame processor. * - * Only supports mute effect for now. + * It is not thread-safe. */ -class AudioFrameProcessor : IFrameProcessor, - IAudioFrameProcessor { +class AudioFrameProcessor( + dispatcher: CoroutineDispatcher, + private val effects: MutableList = mutableListOf() +) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { + private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) + + /** + * Whether the audio is muted. + * + * When the audio is muted, the audio effect are not processed. Only consumer effects are processed. + */ override var isMuted = false private val muteEffect = MuteEffect() - override fun processFrame(frame: RawFrame): RawFrame { - if (isMuted) { - return muteEffect.processFrame(frame) + private fun launchConsumerEffect( + effect: IConsumerAudioEffect, + isMuted: Boolean, + data: RawFrame + ) { + coroutineScope.launch { + val consumeFrame = + data.copy( + rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() + ) + effect.consume(isMuted, consumeFrame) } - return frame } -} \ No newline at end of file + + override fun process(data: RawFrame): RawFrame { + val isMuted = isMuted + + var processedFrame = muteEffect.process(isMuted, data) + + effects.forEach { + if (it is IProcessorAudioEffect) { + processedFrame = it.process(isMuted, processedFrame) + } else if (it is IConsumerAudioEffect) { + launchConsumerEffect(it, isMuted, processedFrame) + } + } + + return processedFrame + } + + override fun close() { + effects.forEach { it.close() } + effects.clear() + + muteEffect.close() + + coroutineScope.cancel() + } + + @Deprecated("'fun toArray(generator: IntFunction!>!): Array<(out) T!>!' is deprecated. This declaration is redundant in Kotlin and might be removed soon.") + @Suppress("DEPRECATION") + override fun toArray(generator: IntFunction?>): Array { + return super.toArray(generator) + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt index 24c7606e1..19cc6ba9c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt @@ -20,7 +20,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio */ interface IAudioFrameProcessor { /** - * Mute audio. + * Whether the processor is muted. */ var isMuted: Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 9560378e1..85ac22509 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -135,12 +135,12 @@ internal class AudioInput( /** * The audio processor. */ - private val frameProcessorInternal = AudioFrameProcessor() - override val processor: IAudioFrameProcessor = frameProcessorInternal + private val processorInternal = AudioFrameProcessor(dispatcherProvider.default) + override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(frameProcessorInternal, config, dispatcherProvider) + PushAudioPort(processorInternal, config, dispatcherProvider) } else { - CallbackAudioPort(frameProcessorInternal) // No threading needed, called from encoder thread + CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } // CONFIG @@ -355,6 +355,15 @@ internal class AudioInput( ) } + try { + processorInternal.close() + } catch (t: Throwable) { + Logger.w( + TAG, + "release: Can't close audio processor: ${t.message}" + ) + } + isStreamingJob.cancel() } coroutineScope.coroutineContext.cancelChildren() @@ -430,7 +439,7 @@ private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProce val timestampInUs = source.fillAudioFrame(buffer) pool.get(buffer, timestampInUs) } - return audioFrameProcessor.processFrame(frame) + return audioFrameProcessor.process(frame) } } From 92b10e79a957f06a10ee368fa6f73d9df0975254 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:45:25 +0100 Subject: [PATCH 14/27] fix(core): processor: duplicate raw buffer dispatching --- .../elements/processing/audio/AudioFrameProcessor.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 99c586a22..f6ccc4934 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.processing.IProcessor import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -49,11 +50,11 @@ class AudioFrameProcessor( isMuted: Boolean, data: RawFrame ) { + val consumeFrame = + data.copy( + rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() + ) coroutineScope.launch { - val consumeFrame = - data.copy( - rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() - ) effect.consume(isMuted, consumeFrame) } } From e768bcfada6defc7ab6e2f1ec6fe543d25cec113 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 10:40:52 +0100 Subject: [PATCH 15/27] refactor(core): rename `clone` to `deepCopy` --- .../endpoints/composites/muxers/mp4/models/TrackChunks.kt | 4 ++-- .../elements/utils/extensions/ByteBufferExtensions.kt | 8 ++++---- .../utils/extensions/ByteBufferExtensionsKtTest.kt | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index cbccabd76..28dde5d47 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -63,7 +63,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.SLCon import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodecConfigurationRecord -import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode @@ -176,7 +176,7 @@ class TrackChunks( } val frameCopy = - frame.copy(rawBuffer = frame.rawBuffer.clone()) // Do not keep mediacodec buffer + frame.copy(rawBuffer = frame.rawBuffer.deepCopy()) // Do not keep mediacodec buffer chunks.last().add(frameId, frameCopy) frameId++ } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index e4beee7ab..88dd17ffc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -253,18 +253,18 @@ fun ByteBuffer.toByteArray(): ByteArray { } /** - * Clone [ByteBuffer]. + * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ -fun ByteBuffer.clone(): ByteBuffer { +fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { - val clone = if (isDirect) { + val copy = if (isDirect) { ByteBuffer.allocateDirect(this.remaining()) } else { ByteBuffer.allocate(this.remaining()) } - return clone.put(this).apply { rewind() } + return copy.put(this).apply { rewind() } } finally { this.position(originalPosition) } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 3ceff1035..489364031 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -117,7 +117,7 @@ class ByteBufferExtensionsKtTest { ) testBuffer.position(2) - val clonedBuffer = testBuffer.clone() + val clonedBuffer = testBuffer.deepCopy() assertArrayEquals( testBuffer.toByteArray(), clonedBuffer.toByteArray() ) From 6a2cf12855eb8c966760d6f1542abb0b62f15a0e Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:14:42 +0100 Subject: [PATCH 16/27] fix(core): processor: consumer audio effect must receive a deep copy of the current audio buffer. Because in callback mode, the rawBuffer belongs to the codec input buffer pool and it could be released before the effect processes the buffer --- .../streampack/core/elements/data/Frame.kt | 20 +++++++++++++++++++ .../elements/processing/RawFramePullPush.kt | 7 ------- .../processing/audio/AudioFrameProcessor.kt | 10 +++++----- .../utils/extensions/ByteBufferExtensions.kt | 20 +++++++++++++++++++ .../elements/utils/pool/ByteBufferPool.kt | 2 +- .../core/pipelines/inputs/AudioInput.kt | 9 +++++++-- 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index c9c08acea..ab615b1ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,7 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.io.Closeable import java.nio.ByteBuffer @@ -36,6 +38,24 @@ interface RawFrame : Closeable { val timestampInUs: Long } +/** + * Deep copy the [RawFrame.rawBuffer] into a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.deepCopy( + bufferPool: IBufferPool, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val copy = this.rawBuffer.deepCopy(bufferPool) + return copy( + rawBuffer = copy, timestampInUs = timestampInUs, onClosed = { + onClosed(it) + bufferPool.put(copy) + } + ) +} /** * Copy a [RawFrame] to a new [RawFrame]. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index eca215f31..aa5bfedd5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -31,13 +31,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean -fun RawFramePullPush( - frameProcessor: IProcessor, - onFrame: suspend (RawFrame) -> Unit, - processDispatcher: CoroutineDispatcher, - isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, ByteBufferPool(isDirect), processDispatcher) - /** * A component that pull a frame from an input and push it to [onFrame] output. * diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index f6ccc4934..ab249b756 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,14 +16,16 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.data.copy +import io.github.thibaultbee.streampack.core.elements.data.deepCopy import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import java.io.Closeable +import java.nio.ByteBuffer import java.util.function.IntFunction /** @@ -32,6 +34,7 @@ import java.util.function.IntFunction * It is not thread-safe. */ class AudioFrameProcessor( + private val bufferPool: IBufferPool, dispatcher: CoroutineDispatcher, private val effects: MutableList = mutableListOf() ) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { @@ -50,10 +53,7 @@ class AudioFrameProcessor( isMuted: Boolean, data: RawFrame ) { - val consumeFrame = - data.copy( - rawBuffer = data.rawBuffer.duplicate().asReadOnlyBuffer() - ) + val consumeFrame = data.deepCopy(bufferPool) coroutineScope.launch { effect.consume(isMuted, consumeFrame) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 88dd17ffc..09cee1ff1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.extensions +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.charset.StandardCharsets @@ -256,6 +257,7 @@ fun ByteBuffer.toByteArray(): ByteArray { * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ +@Deprecated("Use ByteBufferPool instead") fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { @@ -270,6 +272,24 @@ fun ByteBuffer.deepCopy(): ByteBuffer { } } +/** + * Deep copy of [ByteBuffer] from [IBufferPool]. + * + * Don't forget to put the returned [ByteBuffer] to the buffer pool when you are done with it. + * + * @param pool [IBufferPool] to use + * @return [ByteBuffer] deep copy + */ +fun ByteBuffer.deepCopy(pool: IBufferPool): ByteBuffer { + val originalPosition = this.position() + try { + val copy = pool.get(this.remaining()) + return copy.put(this).apply { rewind() } + } finally { + this.position(originalPosition) + } +} + /** * For AVC and HEVC */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt index ce748def0..f3036b9b8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt @@ -87,4 +87,4 @@ class ByteBufferPool(private val isDirect: Boolean) : IBufferPool, C buffers.clear() } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 85ac22509..999bc16e8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameS import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING @@ -132,13 +133,15 @@ internal class AudioInput( } // PROCESSOR + private val bufferPool = ByteBufferPool(true) + /** * The audio processor. */ - private val processorInternal = AudioFrameProcessor(dispatcherProvider.default) + private val processorInternal = AudioFrameProcessor(bufferPool, dispatcherProvider.default) override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(processorInternal, config, dispatcherProvider) + PushAudioPort(processorInternal, config, bufferPool, dispatcherProvider) } else { CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } @@ -390,11 +393,13 @@ private sealed interface IAudioPort : Streamable, Releasable { private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, + bufferPool: ByteBufferPool, dispatcherProvider: IAudioDispatcherProvider ) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, + bufferPool, dispatcherProvider.createAudioDispatcher( 1, THREAD_NAME_AUDIO_PREPROCESSING From 3e1ed7597da606417e12b641e597f95be2877537 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Mon, 2 Feb 2026 12:08:32 +0100 Subject: [PATCH 17/27] fix(core): processor: use `CopyOnWriteArrayList` instead of MutableList to avoid `ConcurrentModificationException` --- .../core/elements/processing/audio/AudioFrameProcessor.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index ab249b756..f10f6af43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -26,17 +26,16 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import java.io.Closeable import java.nio.ByteBuffer +import java.util.concurrent.CopyOnWriteArrayList import java.util.function.IntFunction /** * Audio frame processor. - * - * It is not thread-safe. */ class AudioFrameProcessor( private val bufferPool: IBufferPool, dispatcher: CoroutineDispatcher, - private val effects: MutableList = mutableListOf() + private val effects: CopyOnWriteArrayList = CopyOnWriteArrayList() ) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) From ecb4d15d447259b32c109cfd0c97554c250ff5a5 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:33:55 +0100 Subject: [PATCH 18/27] feat(ui): add a composable preview view --- gradle/libs.versions.toml | 9 ++ settings.libs.gradle.kts | 10 +- ui/{ => compose}/.gitignore | 0 ui/compose/build.gradle.kts | 35 ++++++ ui/{ => compose}/proguard-rules.pro | 0 ui/compose/src/main/AndroidManifest.xml | 1 + .../streampack/compose/PreviewView.kt | 105 ++++++++++++++++++ .../streampack/compose/utils/BitmapUtils.kt | 41 +++++++ ui/ui/.gitignore | 1 + ui/{ => ui}/build.gradle.kts | 0 ui/ui/proguard-rules.pro | 21 ++++ ui/{ => ui}/src/main/AndroidManifest.xml | 0 .../streampack/ui/views/AutoFitSurfaceView.kt | 0 .../streampack/ui/views/PreviewView.kt | 0 .../streampack/ui/views/StreamerExtensions.kt | 0 .../ui/views/VideoSourceExtensions.kt | 0 .../streampack/ui/views/ViewExtensions.kt | 0 ui/{ => ui}/src/main/res/values/attrs.xml | 0 18 files changed, 221 insertions(+), 2 deletions(-) rename ui/{ => compose}/.gitignore (100%) create mode 100644 ui/compose/build.gradle.kts rename ui/{ => compose}/proguard-rules.pro (100%) create mode 100644 ui/compose/src/main/AndroidManifest.xml create mode 100644 ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt create mode 100644 ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt create mode 100644 ui/ui/.gitignore rename ui/{ => ui}/build.gradle.kts (100%) create mode 100644 ui/ui/proguard-rules.pro rename ui/{ => ui}/src/main/AndroidManifest.xml (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt (100%) rename ui/{ => ui}/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt (100%) rename ui/{ => ui}/src/main/res/values/attrs.xml (100%) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ba85fa48b..6ed29f24a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ videoApiClient = "1.6.7" androidxActivity = "1.10.1" androidxAppcompat = "1.7.1" androidxCamera = "1.4.0-alpha13" +androidxComposeBom = "2026.01.00" androidxConstraintlayout = "2.2.1" androidxCore = "1.17.0" androidxDatabinding = "8.13.0" @@ -29,6 +30,7 @@ robolectric = "4.16" komuxer = "0.3.3" srtdroid = "1.9.5" junitKtx = "1.3.0" +compose = "1.10.1" [libraries] android-documentation-plugin = { module = "org.jetbrains.dokka:android-documentation-plugin", version.ref = "dokka" } @@ -42,6 +44,7 @@ android-material = { module = "com.google.android.material:material", version.re androidx-activity = { module = "androidx.activity:activity", version.ref = "androidxActivity" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidxAppcompat" } androidx-camera-viewfinder-view = { module = "androidx.camera.viewfinder:viewfinder-view", version.ref = "androidxCamera" } +androidx-compose-bom = { module = "androidx.compose:compose-bom", version.ref = "androidxComposeBom" } androidx-concurrent-futures = { module = "androidx.concurrent:concurrent-futures", version.ref = "concurrentFutures" } androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "androidxConstraintlayout" } androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } @@ -70,10 +73,16 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re mockk = { module = "io.mockk:mockk", version.ref = "mockk" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } srtdroid-ktx = { module = "io.github.thibaultbee.srtdroid:srtdroid-ktx", version.ref = "srtdroid" } +androidx-compose-runtime = { group = "androidx.compose.runtime", name = "runtime", version.ref = "compose" } +androidx-compose-ui = { group = "androidx.compose.ui", name = "ui", version.ref = "compose" } +androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "compose" } +androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "compose" } +androidx-compose-foundation = { group = "androidx.compose.foundation", name = "foundation", version.ref = "compose" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } android-library = { id = "com.android.library", version.ref = "agp" } +compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } diff --git a/settings.libs.gradle.kts b/settings.libs.gradle.kts index c42addcde..7a8a4328c 100644 --- a/settings.libs.gradle.kts +++ b/settings.libs.gradle.kts @@ -1,11 +1,17 @@ // StreamPack libraries include(":core") project(":core").name = "streampack-core" -include(":ui") -project(":ui").name = "streampack-ui" include(":services") project(":services").name = "streampack-services" +// UI +include(":ui") +project(":ui").projectDir = File(rootDir, "ui/ui") +project(":ui").name = "streampack-ui" +include(":compose") +project(":compose").projectDir = File(rootDir, "ui/compose") +project(":compose").name = "streampack-compose" + // Extensions include(":extension-flv") project(":extension-flv").projectDir = File(rootDir, "extensions/flv") diff --git a/ui/.gitignore b/ui/compose/.gitignore similarity index 100% rename from ui/.gitignore rename to ui/compose/.gitignore diff --git a/ui/compose/build.gradle.kts b/ui/compose/build.gradle.kts new file mode 100644 index 000000000..de8c2b318 --- /dev/null +++ b/ui/compose/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + id(libs.plugins.android.library.get().pluginId) + id(libs.plugins.kotlin.android.get().pluginId) + id("android-library-convention") + alias(libs.plugins.compose.compiler) +} + +description = "Jetpack compose components for StreamPack." + +android { + namespace = "io.github.thibaultbee.streampack.compose" + + defaultConfig { + minSdk = 23 + } + buildFeatures { + compose = true + } +} + +dependencies { + implementation(project(":streampack-core")) + implementation(project(":streampack-ui")) + + val composeBom = platform(libs.androidx.compose.bom) + implementation(composeBom) + implementation(libs.androidx.compose.runtime) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.tooling.preview) + implementation(libs.androidx.compose.foundation) + + androidTestImplementation(composeBom) + + debugImplementation(libs.androidx.compose.ui.tooling) +} diff --git a/ui/proguard-rules.pro b/ui/compose/proguard-rules.pro similarity index 100% rename from ui/proguard-rules.pro rename to ui/compose/proguard-rules.pro diff --git a/ui/compose/src/main/AndroidManifest.xml b/ui/compose/src/main/AndroidManifest.xml new file mode 100644 index 000000000..cc947c567 --- /dev/null +++ b/ui/compose/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt new file mode 100644 index 000000000..379649bb7 --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose + +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.viewinterop.AndroidView +import io.github.thibaultbee.streampack.compose.utils.BitmapUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS +import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource +import io.github.thibaultbee.streampack.core.logger.Logger +import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.ui.views.PreviewView +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.withLock + +private const val TAG = "ComposePreviewView" + +/** + * Displays the preview of a [IWithVideoSource]. + * + * A [IWithVideoSource] must have a video sourc. + * + * @param videoSource the [IWithVideoSource] to preview + * @param modifier the [Modifier] to apply to the [PreviewView] + * @param enableZoomOnPinch enable zoom on pinch gesture + * @param enableTapToFocus enable tap to focus + * @param onTapToFocusTimeoutMs the duration in milliseconds after which the focus area set by tap-to-focus is cleared + */ +@Composable +fun PreviewScreen( + videoSource: IWithVideoSource, + modifier: Modifier = Modifier, + enableZoomOnPinch: Boolean = true, + enableTapToFocus: Boolean = true, + onTapToFocusTimeoutMs: Long = DEFAULT_AUTO_CANCEL_DURATION_MS +) { + val scope = rememberCoroutineScope() + + AndroidView( + factory = { context -> + PreviewView(context).apply { + this.enableZoomOnPinch = enableZoomOnPinch + this.enableTapToFocus = enableTapToFocus + this.onTapToFocusTimeoutMs = onTapToFocusTimeoutMs + + scope.launch { + try { + setVideoSourceProvider(videoSource) + } catch (e: Exception) { + Logger.e(TAG, "Failed to start preview", e) + } + } + } + }, + modifier = modifier, + onRelease = { + scope.launch { + val source = videoSource.videoInput?.sourceFlow?.value as? IPreviewableSource + source?.previewMutex?.withLock { + source.stopPreview() + source.resetPreview() + } + } + }) +} + +@Preview +@Composable +fun PreviewScreenPreview() { + val context = LocalContext.current + val streamer = SingleStreamer(context) + LaunchedEffect(Unit) { + streamer.setVideoSource( + BitmapSourceFactory( + BitmapUtils.createImage( + 1280, + 720 + ) + ) + ) + } + + PreviewScreen(streamer, modifier = Modifier.fillMaxSize()) +} \ No newline at end of file diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt new file mode 100644 index 000000000..c1cfec02b --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose.utils + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint + +object BitmapUtils { + /** + * Creates a bitmap with the given width, height and color. + * + * @param width The width of the bitmap. + * @param height The height of the bitmap. + * @param color The color of the bitmap. + * @return The created bitmap. + */ + fun createImage(width: Int, height: Int, color: Int = Color.RED): Bitmap { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val paint = Paint().apply { + this.color = color + } + canvas.drawRect(0f, 0f, width.toFloat(), height.toFloat(), paint) + return bitmap + } +} diff --git a/ui/ui/.gitignore b/ui/ui/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/ui/ui/.gitignore @@ -0,0 +1 @@ +/build diff --git a/ui/build.gradle.kts b/ui/ui/build.gradle.kts similarity index 100% rename from ui/build.gradle.kts rename to ui/ui/build.gradle.kts diff --git a/ui/ui/proguard-rules.pro b/ui/ui/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/ui/ui/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/ui/src/main/AndroidManifest.xml b/ui/ui/src/main/AndroidManifest.xml similarity index 100% rename from ui/src/main/AndroidManifest.xml rename to ui/ui/src/main/AndroidManifest.xml diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt diff --git a/ui/src/main/res/values/attrs.xml b/ui/ui/src/main/res/values/attrs.xml similarity index 100% rename from ui/src/main/res/values/attrs.xml rename to ui/ui/src/main/res/values/attrs.xml From a83b5ec84e82c290bd0403cc570d17258bb15097 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:12:34 +0100 Subject: [PATCH 19/27] feat(core): camera: add a way to access the TotalCaptureResult --- .../sources/video/camera/CameraSettings.kt | 89 +++++++++++++++---- .../camera/controllers/CameraController.kt | 6 +- .../controllers/CameraSessionController.kt | 11 --- .../camera/utils/CameraSessionCallback.kt | 30 ++++--- .../camera/utils/CaptureResultListener.kt | 28 ++++++ 5 files changed, 123 insertions(+), 41 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 9b0a41b1a..c662fdd21 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -20,6 +20,10 @@ import android.graphics.PointF import android.graphics.Rect import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_ACTIVE_SCAN +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_FOCUSED_LOCKED +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_INACTIVE +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult import android.hardware.camera2.TotalCaptureResult @@ -33,7 +37,6 @@ import androidx.annotation.IntRange import androidx.annotation.RequiresApi import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoFocusModes import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoWhiteBalanceModes @@ -50,6 +53,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.scalerMaxZoom import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.sensitivityRange import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.zoomRatioRange +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.extensions.clamp import io.github.thibaultbee.streampack.core.elements.utils.extensions.isApplicationPortrait import io.github.thibaultbee.streampack.core.elements.utils.extensions.isNormalized @@ -65,8 +69,10 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.CancellationException import java.util.concurrent.atomic.AtomicLong @@ -96,7 +102,9 @@ class CameraSettings internal constructor( } cameraController.addCaptureCallbackListener(captureCallback) awaitClose { - cameraController.removeCaptureCallbackListener(captureCallback) + runBlocking { + cameraController.removeCaptureCallbackListener(captureCallback) + } } }.conflate().distinctUntilChanged() @@ -159,8 +167,6 @@ class CameraSettings internal constructor( val focusMetering = FocusMetering(coroutineScope, characteristics, this, zoom, focus, exposure, whiteBalance) - private val tagBundleFactory = TagBundle.TagBundleFactory() - /** * Directly gets a [CaptureRequest] from the camera. * @@ -184,18 +190,35 @@ class CameraSettings internal constructor( * Applies settings to the camera repeatedly in a synchronized way. * * This method returns when the capture callback is received with the passed request. + * + * @return the total capture result */ - suspend fun applyRepeatingSessionSync() { - val deferred = CompletableDeferred() + suspend fun applyRepeatingSessionSync(): TotalCaptureResult { + val deferred = CompletableDeferred() - val tag = tagBundleFactory.create() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + deferred.complete(result) + return false + } + } + applyRepeatingSession(captureResult) + return deferred.await() + } + + /** + * Applies settings to the camera repeatedly. + * + * @param onCaptureResult the capture result callback. Return `true` to stop the callback. + */ + suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { + val tag = TagBundle.Factory.default.create() val captureCallback = object : CaptureResultListener { override fun onCaptureResult(result: TotalCaptureResult): Boolean { val resultTag = result.request.tag as? TagBundle val keyId = resultTag?.keyId ?: return false if (keyId >= tag.keyId) { - deferred.complete(Unit) - return true + return onCaptureResult.onCaptureResult(result) } return false } @@ -203,7 +226,6 @@ class CameraSettings internal constructor( cameraController.addCaptureCallbackListener(captureCallback) cameraController.setRepeatingSession(tag) - deferred.await() } /** @@ -211,7 +233,7 @@ class CameraSettings internal constructor( */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() - private class TagBundle private constructor(val keyId: Long) { + private class TagBundle(val keyId: Long) { private val tagMap = mutableMapOf().apply { put(TAG_KEY_ID, keyId) } @@ -232,7 +254,7 @@ class CameraSettings internal constructor( * * The purpose is to make sure the tag always contains an increasing id. */ - class TagBundleFactory { + class Factory private constructor() { /** * Next session id. */ @@ -241,6 +263,10 @@ class CameraSettings internal constructor( fun create(): TagBundle { return TagBundle(nextSessionUpdateId.getAndIncrement()) } + + companion object { + val default = Factory() + } } } @@ -1035,7 +1061,35 @@ class CameraSettings internal constructor( ) cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, aeMode) } - cameraSettings.applyRepeatingSession() + + val deferred = CompletableDeferred() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + return when (val afState = result[CaptureResult.CONTROL_AF_STATE]) { + CONTROL_AF_STATE_FOCUSED_LOCKED -> { + deferred.complete(Unit) + true + } + + CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> { + deferred.completeExceptionally(Exception("AF not focused")) + true + } + + CONTROL_AF_STATE_INACTIVE -> { + deferred.completeExceptionally(CancellationException("AF has been cancelled")) + true + } + + CONTROL_AF_STATE_ACTIVE_SCAN -> false + else -> { + deferred.completeExceptionally(IllegalStateException("AF is not in an expected state $afState")) + } + } + } + } + cameraSettings.applyRepeatingSession(captureResult) + deferred.await() } private suspend fun executeMetering( @@ -1052,7 +1106,7 @@ class CameraSettings internal constructor( */ cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, - CameraMetadata.CONTROL_AF_TRIGGER_IDLE + CameraMetadata.CONTROL_AF_TRIGGER_CANCEL ) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { cameraSettings.set( @@ -1063,9 +1117,6 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSessionSync() addFocusMetering(afRectangles, aeRectangles, awbRectangles) - if (afRectangles.isNotEmpty()) { - triggerAf(true) - } // Auto cancel AF trigger after timeoutDurationMs if (timeoutDurationMs > 0) { @@ -1078,6 +1129,10 @@ class CameraSettings internal constructor( } } } + + if (afRectangles.isNotEmpty()) { + triggerAf(true) + } } private suspend fun startFocusAndMetering( diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index ec7000895..a2fca02b0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -21,13 +21,13 @@ import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.util.Range import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineScope @@ -387,7 +387,7 @@ internal class CameraController( * * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ - fun addCaptureCallbackListener(listener: CaptureResultListener) { + suspend fun addCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.addListener(listener) } @@ -396,7 +396,7 @@ internal class CameraController( * * @param listener The listener to remove */ - fun removeCaptureCallbackListener(listener: CaptureResultListener) { + suspend fun removeCaptureCallbackListener(listener: CaptureResultListener) { sessionCallback.removeListener(listener) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index ab193b4ad..b42274eca 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -34,7 +34,6 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -453,14 +452,4 @@ internal class CameraSessionController private constructor( } } } - - interface CaptureResultListener { - /** - * Called when a capture result is received. - * - * @param result The capture result. - * @return true if the listener is finished and should be removed, false otherwise. - */ - fun onCaptureResult(result: TotalCaptureResult): Boolean - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt index ae8a8ad2c..67b8b086d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -19,9 +19,10 @@ import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCaptureSession.CaptureCallback import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraSessionController.CaptureResultListener import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock /** * A capture callback that wraps multiple [CaptureResultListener]. @@ -32,18 +33,23 @@ internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) CaptureCallback() { /* synthetic accessor */ private val resultListeners = mutableSetOf() + private val mutex = Mutex() /** * Adds a capture result listener. * * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. */ - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) + suspend fun addListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.add(listener) + } } - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) + suspend fun removeListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.remove(listener) + } } override fun onCaptureCompleted( @@ -53,14 +59,18 @@ internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) ) { coroutineScope.launch { val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) + mutex.withLock { + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } } } if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) + mutex.withLock { + resultListeners.removeAll(removeSet) + } } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt new file mode 100644 index 000000000..c97eeda2f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.TotalCaptureResult + +interface CaptureResultListener { + /** + * Called when a capture result is received. + * + * @param result The capture result. + * @return true if the listener is finished and should be removed, false otherwise. + */ + fun onCaptureResult(result: TotalCaptureResult): Boolean +} \ No newline at end of file From 841ddc57bf691265e253a5199560cb054f660cbb Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:27:53 +0100 Subject: [PATCH 20/27] fix(core): camera: internalizes constructor of camera settings --- .../sources/video/camera/CameraSettings.kt | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c662fdd21..2310d5c54 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -270,7 +270,7 @@ class CameraSettings internal constructor( } } - class Flash( + class Flash internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -322,7 +322,7 @@ class CameraSettings internal constructor( } } - class WhiteBalance( + class WhiteBalance internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -400,7 +400,7 @@ class CameraSettings internal constructor( } } - class Iso( + class Iso internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -448,7 +448,7 @@ class CameraSettings internal constructor( } } - class ColorCorrection( + class ColorCorrection internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -503,7 +503,7 @@ class CameraSettings internal constructor( } } - class Exposure( + class Exposure internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -679,7 +679,7 @@ class CameraSettings internal constructor( } } - class CropScalerRegionZoom( + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { @@ -754,7 +754,10 @@ class CameraSettings internal constructor( } @RequiresApi(Build.VERSION_CODES.R) - class RZoom(characteristics: CameraCharacteristics, cameraSettings: CameraSettings) : + class RZoom internal constructor( + characteristics: CameraCharacteristics, + cameraSettings: CameraSettings + ) : Zoom(characteristics, cameraSettings) { override val availableRatioRange: Range get() = characteristics.zoomRatioRange @@ -781,7 +784,7 @@ class CameraSettings internal constructor( const val DEFAULT_ZOOM_RATIO = 1f val DEFAULT_ZOOM_RATIO_RANGE = Range(DEFAULT_ZOOM_RATIO, DEFAULT_ZOOM_RATIO) - fun build( + internal fun build( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ): Zoom { @@ -795,7 +798,7 @@ class CameraSettings internal constructor( } - class Focus( + class Focus internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -905,7 +908,7 @@ class CameraSettings internal constructor( } } - class Stabilization( + class Stabilization internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -989,7 +992,7 @@ class CameraSettings internal constructor( } } - class FocusMetering( + class FocusMetering internal constructor( private val coroutineScope: CoroutineScope, private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings, From 453b5b2b9e9195813fdb48dab31258c1bbbd078d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:37:00 +0100 Subject: [PATCH 21/27] feat(core): camera: add an API to set torch strength level --- .../sources/video/camera/CameraSettings.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 2310d5c54..14e7229cc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -320,6 +320,31 @@ class CameraSettings internal constructor( cameraSettings.set(CaptureRequest.FLASH_MODE, mode) cameraSettings.applyRepeatingSession() } + + /** + * Gets the range of supported flash strength. + * Range is from [1-x]. + * + * Use the range to call [setStrengthLevel] + */ + val strengthLevelRange: Range + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + get() = Range( + 1, + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 + ) + + /** + * Sets the flash strength. + * + * @param level flash strength. Range is from [1-x]. + * @see [strengthLevelRange] + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + suspend fun setStrengthLevel(level: Int) { + cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) + cameraSettings.applyRepeatingSession() + } } class WhiteBalance internal constructor( From a77b9faf2efa471185d8ed905af88bdb830ac54d Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:54:07 +0100 Subject: [PATCH 22/27] refactor(core): camera: use lazy initializer for available properties --- .../sources/video/camera/CameraSettings.kt | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index 14e7229cc..c0026d17e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -279,8 +279,7 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, [Boolean.Companion.toString] otherwise. */ - val isAvailable: Boolean - get() = characteristics.isFlashAvailable + val isAvailable: Boolean by lazy { characteristics.isFlashAvailable } /** * Enables or disables flash. @@ -327,12 +326,22 @@ class CameraSettings internal constructor( * * Use the range to call [setStrengthLevel] */ - val strengthLevelRange: Range - @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) - get() = Range( + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val strengthLevelRange: Range by lazy { + Range( 1, characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 ) + } + + val strengthLevel: Int + /** + * Gets the flash strength. + * + * @return the flash strength + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) ?: 1 /** * Sets the flash strength. @@ -356,8 +365,7 @@ class CameraSettings internal constructor( * * @return list of supported white balance modes. */ - val availableAutoModes: List - get() = characteristics.autoWhiteBalanceModes + val availableAutoModes: List by lazy { characteristics.autoWhiteBalanceModes } /** * Gets the auto white balance mode. @@ -388,8 +396,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available white balance metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfWhiteBalanceMeteringRegions + val maxNumOfMeteringRegions: Int by lazy { characteristics.maxNumberOfWhiteBalanceMeteringRegions } /** * Gets the white balance metering regions. @@ -436,8 +443,9 @@ class CameraSettings internal constructor( * * @see [sensorSensitivity] */ - val availableSensorSensitivityRange: Range - get() = characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + val availableSensorSensitivityRange: Range by lazy { + characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + } /** * Gets lens focus distance. @@ -482,11 +490,10 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, `false` otherwise. */ - val isAvailable: Boolean - get() { - return characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] - ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true - } + val isAvailable: Boolean by lazy { + characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] + ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true + } /** * Gets color correction gain. @@ -539,9 +546,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoExposureModes - + val availableAutoModes: List by lazy { characteristics.autoExposureModes } /** * Gets auto exposure mode. @@ -575,9 +580,10 @@ class CameraSettings internal constructor( * @see [availableCompensationStep] * @see [compensation] */ - val availableCompensationRange: Range - get() = characteristics.exposureRange + val availableCompensationRange: Range by lazy { + characteristics.exposureRange ?: DEFAULT_COMPENSATION_RANGE + } /** * Gets current camera exposure compensation step. @@ -591,9 +597,9 @@ class CameraSettings internal constructor( * @see [availableCompensationRange] * @see [compensation] */ - val availableCompensationStep: Rational - get() = characteristics.exposureStep - ?: DEFAULT_COMPENSATION_STEP_RATIONAL + val availableCompensationStep: Rational by lazy { + characteristics.exposureStep ?: DEFAULT_COMPENSATION_STEP_RATIONAL + } /** * Gets exposure compensation. @@ -626,8 +632,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available exposure metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfExposureMeteringRegions + val maxNumOfMeteringRegions by lazy { characteristics.maxNumberOfExposureMeteringRegions } /** * Gets the exposure metering regions. @@ -714,10 +719,11 @@ class CameraSettings internal constructor( private var persistentZoomRatio = 1f private var currentCropRect: Rect? = null - override val availableRatioRange: Range - get() = Range( + override val availableRatioRange: Range by lazy { + Range( DEFAULT_ZOOM_RATIO, characteristics.scalerMaxZoom ) + } override suspend fun getZoomRatio(): Float = mutex.withLock { persistentZoomRatio @@ -784,9 +790,9 @@ class CameraSettings internal constructor( cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { - override val availableRatioRange: Range - get() = characteristics.zoomRatioRange - ?: DEFAULT_ZOOM_RATIO_RANGE + override val availableRatioRange: Range by lazy { + characteristics.zoomRatioRange ?: DEFAULT_ZOOM_RATIO_RANGE + } override suspend fun getZoomRatio(): Float { return cameraSettings.get(CaptureRequest.CONTROL_ZOOM_RATIO) @@ -834,8 +840,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoFocusModes + val availableAutoModes: List by lazy { characteristics.autoFocusModes } /** * Gets the auto focus mode. @@ -868,8 +873,9 @@ class CameraSettings internal constructor( * * @see [lensDistance] */ - val availableLensDistanceRange: Range - get() = characteristics.lensDistanceRange + val availableLensDistanceRange: Range by lazy { + characteristics.lensDistanceRange + } /** * Gets the lens focus distance. @@ -902,9 +908,9 @@ class CameraSettings internal constructor( /** * Get maximum number of available focus metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfFocusMeteringRegions - ?: DEFAULT_MAX_NUM_OF_METERING_REGION + val maxNumOfMeteringRegions: Int by lazy { + characteristics.maxNumberOfFocusMeteringRegions ?: DEFAULT_MAX_NUM_OF_METERING_REGION + } /** * Gets the focus metering regions. @@ -977,8 +983,9 @@ class CameraSettings internal constructor( * * @see [isEnableOptical] */ - val isOpticalAvailable: Boolean - get() = characteristics.isOpticalStabilizationAvailable + val isOpticalAvailable: Boolean by lazy { + characteristics.isOpticalStabilizationAvailable + } /** From a76c80b83d5b2a5926d919d38c0f6b015e98875a Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:41:29 +0100 Subject: [PATCH 23/27] refactor(*): add listener for camera zoom --- .../sources/video/camera/CameraSettings.kt | 42 +++++++++++++++- .../streampack/app/ui/main/PreviewFragment.kt | 11 ++-- .../app/ui/main/PreviewViewModel.kt | 2 +- .../streampack/ui/views/PreviewView.kt | 50 +++++++++++++++---- 4 files changed, 88 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index c0026d17e..d81e54fb4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -73,6 +73,7 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.CancellationException +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicLong @@ -684,6 +685,8 @@ class CameraSettings internal constructor( protected val characteristics: CameraCharacteristics, protected val cameraSettings: CameraSettings ) { + private val listeners: CopyOnWriteArrayList = CopyOnWriteArrayList() + abstract val availableRatioRange: Range internal abstract suspend fun getCropSensorRegion(): Rect @@ -709,6 +712,20 @@ class CameraSettings internal constructor( } } + fun addListener(listener: OnZoomChangedListener) { + listeners.add(listener) + } + + fun removeListener(listener: OnZoomChangedListener) { + listeners.remove(listener) + } + + protected fun notifyZoomListeners(zoomRatio: Float) { + listeners.forEach { + it.onZoomChanged(zoomRatio) + } + } + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings @@ -732,6 +749,11 @@ class CameraSettings internal constructor( override suspend fun setZoomRatio(zoomRatio: Float) { mutex.withLock { val clampedValue = zoomRatio.clamp(availableRatioRange) + if (clampedValue == persistentZoomRatio) { + return@withLock + } + persistentZoomRatio = clampedValue + currentCropRect = getCropRegion( characteristics, clampedValue @@ -739,8 +761,8 @@ class CameraSettings internal constructor( cameraSettings.set( CaptureRequest.SCALER_CROP_REGION, currentCropRect ) - cameraSettings.applyRepeatingSession() - persistentZoomRatio = clampedValue + cameraSettings.applyRepeatingSessionSync() + notifyZoomListeners(clampedValue) } } @@ -800,10 +822,14 @@ class CameraSettings internal constructor( } override suspend fun setZoomRatio(zoomRatio: Float) { + if (zoomRatio == getZoomRatio()) { + return + } cameraSettings.set( CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio.clamp(availableRatioRange) ) cameraSettings.applyRepeatingSession() + notifyZoomListeners(zoomRatio) } override suspend fun getCropSensorRegion(): Rect { @@ -826,6 +852,18 @@ class CameraSettings internal constructor( } } } + + /** + * Listener for zoom change. + */ + interface OnZoomChangedListener { + /** + * Called when the zoom ratio changes. + * + * @param zoomRatio the zoom ratio + */ + fun onZoomChanged(zoomRatio: Float) + } } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index acd8f2a14..a7d2d9aca 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -34,6 +34,7 @@ import io.github.thibaultbee.streampack.app.R import io.github.thibaultbee.streampack.app.databinding.MainFragmentBinding import io.github.thibaultbee.streampack.app.utils.DialogUtils import io.github.thibaultbee.streampack.app.utils.PermissionManager +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerViewModelLifeCycleObserver @@ -182,15 +183,17 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { private fun inflateStreamerPreview(streamer: IWithVideoSource) { val preview = binding.preview // Set camera settings button when camera is started - preview.listener = object : PreviewView.Listener { + preview.listener = object : PreviewView.PreviewListener { override fun onPreviewStarted() { Log.i(TAG, "Preview started") } + } - override fun onZoomRationOnPinchChanged(zoomRatio: Float) { - previewViewModel.onZoomRationOnPinchChanged() + preview.setZoomListener(object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + previewViewModel.onZoomChanged() } - } + }) // Wait till streamer exists to set it to the SurfaceView. lifecycleScope.launch { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index bf57eed0c..cf15308d4 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -247,7 +247,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - fun onZoomRationOnPinchChanged() { + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 88dc5a1eb..5cdbfe83b 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -35,6 +35,7 @@ import androidx.camera.viewfinder.core.ScaleType import androidx.camera.viewfinder.core.ViewfinderSurfaceRequest import androidx.camera.viewfinder.core.populateFromCharacteristics import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.getCameraCharacteristics @@ -108,9 +109,11 @@ class PreviewView @JvmOverloads constructor( } /** - * The [Listener] to listen to specific view events. + * The [PreviewListener] to listen to specific view events. */ - var listener: Listener? = null + var listener: PreviewListener? = null + + private var zoomListener: CameraSettings.Zoom.OnZoomChangedListener? = null private var touchUpEvent: MotionEvent? = null @@ -176,12 +179,46 @@ class PreviewView @JvmOverloads constructor( } if (newVideoSource is IPreviewableSource) { attachToStreamerIfReady(true) + zoomListener?.let { + registerZoomListener(it) + } } } } } } + /** + * Sets the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + * + * @param listener the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + */ + fun setZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener?) { + if (listener == null) { + unregisterZoomListener() + } else { + registerZoomListener(listener) + } + zoomListener = listener + } + + private fun registerZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener) { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.addListener(listener) + } + } + + private fun unregisterZoomListener() { + zoomListener?.let { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.removeListener(it) + } + } + } + + /** * Sets the [IWithVideoSource] to preview. * @@ -504,7 +541,6 @@ class PreviewView @JvmOverloads constructor( mutex.withLock { val zoom = source.settings.zoom zoom.onPinch(scaleFactor) - listener?.onZoomRationOnPinchChanged(zoom.getZoomRatio()) } } return true @@ -518,7 +554,7 @@ class PreviewView @JvmOverloads constructor( /** * A listener for the [PreviewView]. */ - interface Listener { + interface PreviewListener { /** * Called when the preview is started. */ @@ -528,12 +564,6 @@ class PreviewView @JvmOverloads constructor( * Called when the preview failed to start. */ fun onPreviewFailed(t: Throwable) {} - - /** - * Called when the zoom ratio is changed. - * @param zoomRatio the new zoom ratio - */ - fun onZoomRationOnPinchChanged(zoomRatio: Float) {} } /** From 6267632bf563503dfb8d614a04e6dcc9b3c39744 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Tue, 3 Feb 2026 15:50:13 +0100 Subject: [PATCH 24/27] fix(core): camera: prefix torch strength level range with available --- .../core/elements/sources/video/camera/CameraSettings.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index d81e54fb4..91fc4f7b9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -328,7 +328,7 @@ class CameraSettings internal constructor( * Use the range to call [setStrengthLevel] */ @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) - val strengthLevelRange: Range by lazy { + val availableStrengthLevelRange: Range by lazy { Range( 1, characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 @@ -348,7 +348,7 @@ class CameraSettings internal constructor( * Sets the flash strength. * * @param level flash strength. Range is from [1-x]. - * @see [strengthLevelRange] + * @see [availableStrengthLevelRange] */ @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) suspend fun setStrengthLevel(level: Int) { From c5a5632a2d297d7f7ef235b720a6ed7c8948a5f3 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:44:36 +0100 Subject: [PATCH 25/27] feat(core): make SingleStreamer inputs non nullable --- .../core/pipelines/StreamerPipelineTest.kt | 4 +- .../file/CameraSingleStreamerFileTest.kt | 1 + .../CameraSingleStreamerMultiEndpointTest.kt | 1 + .../state/CameraSingleStreamerStateTest.kt | 1 + .../single/state/SingleStreamerStateTest.kt | 1 + .../core/streamer/utils/StreamerUtils.kt | 4 +- .../streampack/core/interfaces/ISource.kt | 25 +- .../core/pipelines/StreamerPipeline.kt | 6 +- .../single/AudioOnlySingleStreamer.kt | 31 +- .../core/streamers/single/ISingleStreamer.kt | 27 +- .../core/streamers/single/SingleStreamer.kt | 330 +++++------------- .../streamers/single/SingleStreamerImpl.kt | 278 +++++++++++++++ .../single/VideoOnlySingleStreamer.kt | 56 ++- .../app/ui/main/PreviewViewModel.kt | 53 ++- .../ui/main/usecases/BuildStreamerUseCase.kt | 41 ++- .../streampack/app/utils/Extensions.kt | 4 +- .../services/MediaProjectionService.kt | 2 +- .../services/utils/StreamerFactory.kt | 22 +- .../streampack/ui/views/PreviewView.kt | 6 +- .../streampack/ui/views/StreamerExtensions.kt | 4 +- 20 files changed, 560 insertions(+), 337 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt index 8de63b080..d8591b2a3 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt @@ -83,8 +83,8 @@ class StreamerPipelineTest { ) streamerPipeline.addOutput(output) - val audioSource = streamerPipeline.audioInput?.sourceFlow?.value as IAudioSourceInternal - val videoSource = streamerPipeline.videoInput?.sourceFlow?.value as IVideoSourceInternal + val audioSource = streamerPipeline.audioInput.sourceFlow.value as IAudioSourceInternal + val videoSource = streamerPipeline.videoInput.sourceFlow.value as IVideoSourceInternal streamerPipeline.startStream() assertTrue(streamerPipeline.isStreamingFlow.first { it }) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 548b4e053..607d4849e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.single.utils.SingleStreame import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt index 9a1f8a278..34da1422f 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt @@ -27,6 +27,7 @@ import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt index 0f5fcebd7..ee9232fe6 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt @@ -30,6 +30,7 @@ import io.github.thibaultbee.streampack.core.streamer.surface.SurfaceViewTestAct import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt index bfe9a2e05..81387d36b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import kotlinx.coroutines.test.runTest import org.junit.After diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt index 6571955c4..d4835c14e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt @@ -4,7 +4,7 @@ import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.Media import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer -import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext @@ -14,7 +14,7 @@ import kotlin.time.Duration.Companion.seconds object StreamerUtils { suspend fun runSingleStream( - streamer: SingleStreamer, + streamer: ISingleStreamer, descriptor: MediaDescriptor, duration: Duration, pollDuration: Duration = 1.seconds diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt index 60a476805..80ae9b9e4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt @@ -36,7 +36,7 @@ interface IWithAudioSource { /** * The audio input to access to advanced settings. */ - val audioInput: IAudioInput? + val audioInput: IAudioInput /** * Sets a new audio source. @@ -44,8 +44,7 @@ interface IWithAudioSource { * @param audioSourceFactory The new audio source factory. */ suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) { - val audio = requireNotNull(audioInput) { "Audio input is not available" } - audio.setSource(audioSourceFactory) + audioInput.setSource(audioSourceFactory) } } @@ -63,9 +62,9 @@ interface IWithVideoRotation { */ interface IWithVideoSource { /** - * The audio input to access to advanced settings. + * The video input to access to advanced settings. */ - val videoInput: IVideoInput? + val videoInput: IVideoInput /** * Sets the video source. @@ -73,8 +72,7 @@ interface IWithVideoSource { * The previous video source will be released unless its preview is still running. */ suspend fun setVideoSource(videoSourceFactory: IVideoSourceInternal.Factory) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(videoSourceFactory) + videoInput.setSource(videoSourceFactory) } } @@ -87,8 +85,7 @@ interface IWithVideoSource { */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setCameraId(cameraId: String) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(CameraSourceFactory(cameraId)) + videoInput.setSource(CameraSourceFactory(cameraId)) } @@ -96,7 +93,7 @@ suspend fun IWithVideoSource.setCameraId(cameraId: String) { * Whether the video source has a preview. */ val IWithVideoSource.isPreviewable: Boolean - get() = videoInput?.sourceFlow?.value is IPreviewableSource + get() = videoInput.sourceFlow?.value is IPreviewableSource /** * Sets the preview surface. @@ -105,7 +102,7 @@ val IWithVideoSource.isPreviewable: Boolean * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.setPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.setPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.setPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -142,7 +139,7 @@ suspend fun IWithVideoSource.setPreview(textureView: TextureView) = * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.startPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview() ?: throw IllegalStateException("Video source is not previewable") } @@ -154,7 +151,7 @@ suspend fun IWithVideoSource.startPreview() { * @see [IWithVideoSource.stopPreview] */ suspend fun IWithVideoSource.startPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -195,5 +192,5 @@ suspend fun IWithVideoSource.startPreview(textureView: TextureView) = * Stops video preview. */ suspend fun IWithVideoSource.stopPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.stopPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.stopPreview() } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index 26d417892..28e53f39e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -116,7 +116,8 @@ open class StreamerPipeline( } else { null } - override val audioInput: IAudioInput? = _audioInput + override val audioInput: IAudioInput + get() = requireNotNull(_audioInput) { "Audio input is not available" } private val _videoInput = if (withVideo) { VideoInput(context, surfaceProcessorFactory, dispatcherProvider) { @@ -126,7 +127,8 @@ open class StreamerPipeline( null } - override val videoInput: IVideoInput? = _videoInput + override val videoInput: IVideoInput + get() = requireNotNull(_videoInput) { "Video input is not available" } private val _isStreamingFlow = MutableStateFlow(false) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt index d8043700c..2408dda37 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt @@ -25,8 +25,9 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput -import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo /** @@ -35,15 +36,18 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param context the application context * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If parameter is null, no audio source are set. It can be set later with [AudioOnlySingleStreamer.setAudioSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun AudioOnlySingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory = MicrophoneSourceFactory(), - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): AudioOnlySingleStreamer { val streamer = AudioOnlySingleStreamer( context = context, endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) return streamer @@ -54,16 +58,19 @@ suspend fun AudioOnlySingleStreamer( * * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class AudioOnlySingleStreamer( context: Context, - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IAudioSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = true, - withVideo = false + withVideo = false, + endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow @@ -74,8 +81,8 @@ class AudioOnlySingleStreamer( get() = streamer.info override val audioConfigFlow = streamer.audioConfigFlow - override val audioInput: IAudioInput = streamer.audioInput!! - + override val audioInput: IAudioInput = streamer.audioInput + override val audioEncoder: IEncoder? get() = streamer.audioEncoder @@ -97,12 +104,4 @@ class AudioOnlySingleStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() - - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - throw UnsupportedOperationException("Audio single streamer does not support bitrate regulator controller") - } - - override fun removeBitrateRegulatorController() { - // Do nothing - } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt index d19ffbfc3..f0d14fe8c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt @@ -55,22 +55,25 @@ interface ISingleStreamer : IOpenableStreamer { * Gets configuration information */ fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo +} +val ISingleStreamer.withAudio: Boolean /** - * Adds a bitrate regulator controller to the streamer. + * Whether the streamer has an audio source. */ - fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + get() = this is IAudioSingleStreamer +val ISingleStreamer.withVideo: Boolean /** - * Removes the bitrate regulator controller from the streamer. + * Whether the streamer has a video source. */ - fun removeBitrateRegulatorController() -} + get() = this is IVideoSingleStreamer + /** * An audio single Streamer */ -interface IAudioSingleStreamer : IAudioStreamer { +interface IAudioSingleStreamer : IAudioStreamer, ISingleStreamer { /** * The audio configuration flow. */ @@ -85,7 +88,7 @@ interface IAudioSingleStreamer : IAudioStreamer { /** * A video single streamer. */ -interface IVideoSingleStreamer : IVideoStreamer { +interface IVideoSingleStreamer : IVideoStreamer, ISingleStreamer { /** * The video configuration flow. */ @@ -95,5 +98,15 @@ interface IVideoSingleStreamer : IVideoStreamer { * Advanced settings for the video encoder. */ val videoEncoder: IEncoder? + + /** + * Adds a bitrate regulator controller to the streamer. + */ + fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + + /** + * Removes the bitrate regulator controller from the streamer. + */ + fun removeBitrateRegulatorController() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt index d7c1138da..6b3caa9eb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt @@ -25,7 +25,6 @@ import androidx.annotation.RequiresApi import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -35,7 +34,6 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MediaProjectionAudioSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue @@ -43,19 +41,10 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController -import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo -import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraSingleStreamer( @@ -73,10 +64,16 @@ suspend fun cameraSingleStreamer( cameraId: String = context.defaultCameraId, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( - context, withAudio = true, withVideo = true, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) if (audioSourceFactory != null) { @@ -92,20 +89,24 @@ suspend fun cameraSingleStreamer( * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -121,20 +122,24 @@ suspend fun audioVideoMediaProjectionSingleStreamer( * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -152,6 +157,8 @@ suspend fun videoMediaProjectionSingleStreamer( * @param videoSourceFactory the video source factory. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun SingleStreamer( context: Context, @@ -159,13 +166,15 @@ suspend fun SingleStreamer( videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, - withAudio = true, - withVideo = true, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) @@ -173,249 +182,102 @@ suspend fun SingleStreamer( } /** - * A [ISingleStreamer] implementation for audio and video. + * A [ISingleStreamer] implementation for both audio and video. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class SingleStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class SingleStreamer( + context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider(), ) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { - private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, - surfaceProcessorFactory, - dispatcherProvider + private val streamer = SingleStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - private val pipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, - withVideo, - endpointFactory, - defaultRotation - ) as IEncodingPipelineOutputInternal - } - - override val throwableFlow: StateFlow = - merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - override val isOpenFlow: StateFlow - get() = pipelineOutput.isOpenFlow - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // AUDIO - /** - * The audio input. - * It allows advanced audio source settings. - */ - override val audioInput = pipeline.audioInput + override val endpoint: IEndpoint + get() = streamer.endpoint + override val info: IConfigurationInfo + get() = streamer.info + override val audioConfigFlow = streamer.audioConfigFlow override val audioEncoder: IEncoder? - get() = pipelineOutput.audioEncoder - - override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = - pipeline.setAudioSource(audioSourceFactory) - - // VIDEO - /** - * The video input. - * It allows advanced video source settings. - */ - override val videoInput = pipeline.videoInput + get() = streamer.audioEncoder + override val audioInput: IAudioInput = streamer.audioInput!! + override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? - get() = pipelineOutput.videoEncoder - - // ENDPOINT - override val endpoint: IEndpoint - get() = pipelineOutput.endpoint + get() = streamer.videoEncoder + override val videoInput: IVideoInput = streamer.videoInput!! /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } - - /** - * Gets configuration information. - * - * Could throw an exception if the endpoint needs to infer the configuration from the - * [MediaDescriptor]. - * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. - */ - override val info: IConfigurationInfo - get() = if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpoint.info) - } else { - StreamerConfigurationInfo(endpoint.info) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Gets configuration information from [MediaDescriptor]. - * - * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is - * already known. - * - * @param descriptor the media descriptor - */ - override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { - val endpointInfo = try { - endpoint.info - } catch (_: Throwable) { - endpoint.getInfo(descriptor) - } - return if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpointInfo) - } else { - StreamerConfigurationInfo(endpointInfo) - } - } - - // CONFIGURATION - /** - * The audio configuration flow. - */ - override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow - - /** - * Configures audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * @param audioConfig Audio configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: AudioConfig) { - pipelineOutput.setAudioCodecConfig(audioConfig) - } + override suspend fun setAudioConfig(audioConfig: AudioConfig) = + streamer.setAudioConfig(audioConfig) - /** - * The video configuration flow. - */ - override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow - - /** - * Configures video settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ - override suspend fun setVideoConfig(videoConfig: VideoConfig) { - pipelineOutput.setVideoCodecConfig(videoConfig) - } + override suspend fun setVideoConfig(videoConfig: VideoConfig) = + streamer.setVideoConfig(videoConfig) - /** - * Configures both video and audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [IStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } + override fun getInfo(descriptor: MediaDescriptor) = streamer.getInfo(descriptor) - /** - * Opens the streamer endpoint. - * - * @param descriptor Media descriptor to open - */ - override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + override suspend fun open(descriptor: MediaDescriptor) = streamer.open(descriptor) - /** - * Closes the streamer endpoint. - */ - override suspend fun close() = pipelineOutput.close() + override suspend fun close() = streamer.close() - /** - * Starts audio/video stream. - * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote - * device. - * To avoid creating an unresponsive UI, do not call on main thread. - * - * @see [stopStream] - */ - override suspend fun startStream() = pipelineOutput.startStream() + override suspend fun startStream() = streamer.startStream() - /** - * Stops audio/video stream. - * - * Internally, it resets audio and video recorders and encoders to get them ready for another - * [startStream] session. It explains why preview could be restarted. - * - * @see [startStream] - */ - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun stopStream() = streamer.stopStream() - /** - * Releases the streamer. - */ - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } + override suspend fun release() = streamer.release() - /** - * Adds a bitrate regulator controller. - * - * Limitation: it is only available for SRT for now. - */ override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = - pipelineOutput.addBitrateRegulatorController(controllerFactory) + streamer.addBitrateRegulatorController(controllerFactory) - /** - * Removes the bitrate regulator controller. - */ - override fun removeBitrateRegulatorController() = - pipelineOutput.removeBitrateRegulatorController() + override fun removeBitrateRegulatorController() = streamer.removeBitrateRegulatorController() +} - companion object { - const val TAG = "SingleStreamer" - } + +/** + * Configures both video and audio settings. + * It is the first method to call after a [SingleStreamer] instantiation. + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [SingleStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun SingleStreamer.setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt new file mode 100644 index 000000000..e334ca746 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.single + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor +import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController +import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + +/** + * A [ISingleStreamer] implementation for audio and video. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + */ +internal class SingleStreamerImpl( + private val context: Context, + withAudio: Boolean, + withVideo: Boolean, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { + private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, + surfaceProcessorFactory, + dispatcherProvider + ) + private val pipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, + withVideo, + endpointFactory, + defaultRotation + ) as IEncodingPipelineOutputInternal + } + + override val throwableFlow: StateFlow = + merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + override val isOpenFlow: StateFlow + get() = pipelineOutput.isOpenFlow + + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + // AUDIO + /** + * The audio input. + * It allows advanced audio source settings. + */ + override val audioInput: IAudioInput + get() = pipeline.audioInput + + override val audioEncoder: IEncoder? + get() = pipelineOutput.audioEncoder + + override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = + pipeline.setAudioSource(audioSourceFactory) + + // VIDEO + /** + * The video input. + * It allows advanced video source settings. + */ + override val videoInput: IVideoInput + get() = pipeline.videoInput + + override val videoEncoder: IEncoder? + get() = pipelineOutput.videoEncoder + + // ENDPOINT + override val endpoint: IEndpoint + get() = pipelineOutput.endpoint + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Gets configuration information. + * + * Could throw an exception if the endpoint needs to infer the configuration from the + * [MediaDescriptor]. + * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. + */ + override val info: IConfigurationInfo + get() = if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpoint.info) + } else { + StreamerConfigurationInfo(endpoint.info) + } + + /** + * Gets configuration information from [MediaDescriptor]. + * + * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is + * already known. + * + * @param descriptor the media descriptor + */ + override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { + val endpointInfo = try { + endpoint.info + } catch (_: Throwable) { + endpoint.getInfo(descriptor) + } + return if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpointInfo) + } else { + StreamerConfigurationInfo(endpointInfo) + } + } + + // CONFIGURATION + /** + * The audio configuration flow. + */ + override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow + + /** + * Configures audio settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and audio capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * @param audioConfig Audio configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: AudioConfig) { + pipelineOutput.setAudioCodecConfig(audioConfig) + } + + /** + * The video configuration flow. + */ + override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow + + /** + * Configures video settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + override suspend fun setVideoConfig(videoConfig: VideoConfig) { + pipelineOutput.setVideoCodecConfig(videoConfig) + } + + /** + * Opens the streamer endpoint. + * + * @param descriptor Media descriptor to open + */ + override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + + /** + * Closes the streamer endpoint. + */ + override suspend fun close() = pipelineOutput.close() + + /** + * Starts audio/video stream. + * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote + * device. + * To avoid creating an unresponsive UI, do not call on main thread. + * + * @see [stopStream] + */ + override suspend fun startStream() = pipelineOutput.startStream() + + /** + * Stops audio/video stream. + * + * Internally, it resets audio and video recorders and encoders to get them ready for another + * [startStream] session. It explains why preview could be restarted. + * + * @see [startStream] + */ + override suspend fun stopStream() = pipeline.stopStream() + + /** + * Releases the streamer. + */ + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } + + /** + * Adds a bitrate regulator controller. + * + * Limitation: it is only available for SRT for now. + */ + override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = + pipelineOutput.addBitrateRegulatorController(controllerFactory) + + /** + * Removes the bitrate regulator controller. + */ + override fun removeBitrateRegulatorController() = + pipelineOutput.removeBitrateRegulatorController() + + companion object Companion { + const val TAG = "SingleStreamer" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt index fa8071750..c224ba146 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt @@ -26,12 +26,16 @@ import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -44,39 +48,53 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param cameraId the camera id to use. By default, it is the default camera. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlySingleStreamer( context: Context, cameraId: String = context.defaultCameraId, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( - context, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer } /** - * Creates a [SingleStreamer] with the screen as video source and no audio source. + * Creates a [VideoOnlySingleStreamer] with the screen as video source and no audio source. * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlySingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -89,17 +107,23 @@ suspend fun videoMediaProjectionVideoOnlySingleStreamer( * @param videoSourceFactory the video source factory. If parameter is null, no audio source are set. It can be set later with [VideoOnlySingleStreamer.setVideoSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlySingleStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -111,19 +135,27 @@ suspend fun VideoOnlySingleStreamer( * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlySingleStreamer( context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IVideoSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + endpointFactory = endpointFactory, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow @@ -136,7 +168,7 @@ class VideoOnlySingleStreamer( override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? get() = streamer.videoEncoder - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index cf15308d4..9dcb6c070 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -55,7 +55,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio +import io.github.thibaultbee.streampack.core.streamers.single.withVideo import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrtBitrateRegulatorController import kotlinx.coroutines.CancellationException @@ -79,16 +84,18 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod private val defaultDispatcher = Dispatchers.IO - private val buildStreamerUseCase = BuildStreamerUseCase(application, storageRepository) + private val buildStreamerUseCase = BuildStreamerUseCase(application) private val streamerFlow = - MutableStateFlow( - SingleStreamer( - application, - runBlocking { storageRepository.isAudioEnableFlow.first() }) // TODO avoid runBlocking + MutableStateFlow( + buildFirstStreamer(runBlocking { storageRepository.isAudioEnableFlow.first() }) ) - private val streamer: SingleStreamer + + private val streamer: IVideoSingleStreamer get() = streamerFlow.value + private val audioStreamer: IAudioSingleStreamer? + get() = streamer as? IAudioSingleStreamer? + val streamerLiveData = streamerFlow.asLiveData() /** @@ -111,10 +118,10 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val requiredPermissions: List get() { val permissions = mutableListOf() - if (streamer.videoInput?.sourceFlow is ICameraSource) { + if (streamer.videoInput.sourceFlow is ICameraSource) { permissions.add(Manifest.permission.CAMERA) } - if (streamer.audioInput?.sourceFlow?.value is IAudioRecordSource) { + if (audioStreamer?.audioInput?.sourceFlow?.value is IAudioRecordSource) { permissions.add(Manifest.permission.RECORD_AUDIO) } storageRepository.endpointDescriptorFlow.asLiveData().value?.let { @@ -149,7 +156,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // Set audio source and video source if (streamer.withAudio) { Log.i(TAG, "Audio source is enabled. Setting audio source") - streamer.setAudioSource(MicrophoneSourceFactory()) + audioStreamer!!.setAudioSource(MicrophoneSourceFactory()) } else { Log.i(TAG, "Audio source is disabled") } @@ -167,7 +174,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // TODO: cancel jobs linked to previous streamer viewModelScope.launch { - streamer.videoInput?.sourceFlow?.collect { + streamer.videoInput.sourceFlow.collect { notifySourceChanged() } } @@ -226,7 +233,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod ) == PackageManager.PERMISSION_GRANTED ) { try { - streamer.setAudioConfig(config) + audioStreamer!!.setAudioConfig(config) } catch (t: Throwable) { Log.e(TAG, "setAudioConfig failed", t) _streamerErrorLiveData.postValue("setAudioConfig: ${t.message ?: "Unknown error"}") @@ -247,6 +254,14 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } + private fun buildFirstStreamer(isAudioEnable: Boolean): IVideoSingleStreamer { + return if (isAudioEnable) { + SingleStreamer(application) + } else { + VideoOnlySingleStreamer(application) + } + } + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } @@ -255,7 +270,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun configureAudio() { viewModelScope.launch { try { - storageRepository.audioConfigFlow.first()?.let { streamer.setAudioConfig(it) } + storageRepository.audioConfigFlow.first()?.let { audioStreamer?.setAudioConfig(it) } ?: Log.i( TAG, "Audio is disabled" @@ -271,7 +286,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun initializeVideoSource() { viewModelScope.launch { videoSourceMutex.withLock { - if (streamer.videoInput?.sourceFlow?.value == null) { + if (streamer.videoInput.sourceFlow.value == null) { streamer.setVideoSource(CameraSourceFactory(defaultCameraId)) } else { Log.i(TAG, "Camera source already set") @@ -327,7 +342,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } fun setMute(isMuted: Boolean) { - streamer.audioInput?.isMuted = isMuted + audioStreamer?.audioInput?.isMuted = isMuted } @RequiresPermission(Manifest.permission.CAMERA) @@ -337,7 +352,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod * exception instead of crashing. You can either catch the exception or check if the * configuration is valid for the new camera with [Context.isFpsSupported]. */ - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { @@ -357,7 +372,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod */ viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { streamer.setNextCameraId(application) } @@ -369,7 +384,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun toggleVideoSource() { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value val nextSource = when (videoSource) { is ICameraSource -> { BitmapSourceFactory(testBitmap) @@ -390,7 +405,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - val isCameraSource = streamer.videoInput?.sourceFlow?.map { it is ICameraSource }?.asLiveData() + val isCameraSource = streamer.videoInput.sourceFlow?.map { it is ICameraSource }?.asLiveData() val isFlashAvailable = MutableLiveData(false) fun toggleFlash() { @@ -514,7 +529,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } private fun notifySourceChanged() { - val videoSource = streamer.videoInput?.sourceFlow?.value ?: return + val videoSource = streamer.videoInput.sourceFlow.value ?: return if (videoSource is ICameraSource) { notifyCameraChanged(videoSource) } else { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt index 417f6614c..beef3a8f9 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt @@ -4,15 +4,17 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import androidx.core.app.ActivityCompat -import io.github.thibaultbee.streampack.app.data.storage.DataStoreRepository +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio class BuildStreamerUseCase( - private val context: Context, - private val dataStoreRepository: DataStoreRepository + private val context: Context ) { /** - * Build a new [SingleStreamer] based on audio and video preferences. + * Build a new [IVideoSingleStreamer] based on audio and video preferences. * * Only create a new streamer if the previous one is not the same type. * @@ -21,28 +23,37 @@ class BuildStreamerUseCase( * @param previousStreamer Previous streamer to check if we need to create a new one. */ suspend operator fun invoke( - previousStreamer: SingleStreamer, + previousStreamer: IVideoSingleStreamer, isAudioEnable: Boolean - ): SingleStreamer { + ): IVideoSingleStreamer { if (previousStreamer.withAudio != isAudioEnable) { - return SingleStreamer(context, isAudioEnable).apply { - // Get previous streamer config if any + previousStreamer.release() + + val streamer = if (isAudioEnable) { + SingleStreamer(context) + } else { + VideoOnlySingleStreamer(context) + } + + // Get previous streamer config if any + if ((previousStreamer is IAudioSingleStreamer) && (streamer is IAudioSingleStreamer)) { val audioConfig = previousStreamer.audioConfigFlow.value - val videoConfig = previousStreamer.videoConfigFlow.value - if ((audioConfig != null && isAudioEnable)) { + if (audioConfig != null) { if (ActivityCompat.checkSelfPermission( context, Manifest.permission.RECORD_AUDIO ) == PackageManager.PERMISSION_GRANTED ) { - setAudioConfig(audioConfig) + streamer.setAudioConfig(audioConfig) } } - if (videoConfig != null) { - setVideoConfig(videoConfig) - } - previousStreamer.release() } + + val videoConfig = previousStreamer.videoConfigFlow.value + if (videoConfig != null) { + streamer.setVideoConfig(videoConfig) + } + return streamer } return previousStreamer } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt index ec45c6943..bbe54ba7d 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt @@ -37,7 +37,7 @@ import io.github.thibaultbee.streampack.core.interfaces.setCameraId @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setNextCameraId(context: Context) { val cameras = context.cameraManager.cameras - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val newCameraId = if (videoSource is ICameraSource) { val currentCameraIndex = cameras.indexOf(videoSource.cameraId) @@ -52,7 +52,7 @@ suspend fun IWithVideoSource.setNextCameraId(context: Context) { @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.toggleBackToFront(context: Context) { val cameraManager = context.cameraManager - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val cameras = if (videoSource is ICameraSource) { if (cameraManager.isBackCamera(videoSource.cameraId)) { cameraManager.frontCameras diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt index 0b740e6b5..5f5888bfe 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt @@ -163,7 +163,7 @@ abstract class MediaProjectionService( } if (streamer is IWithVideoSource) { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is IMediaProjectionSource) { streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) } else if (videoSource == null) { diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 61f5bc703..78e08fbab 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -16,13 +16,16 @@ package io.github.thibaultbee.streampack.services.utils import android.content.Context +import android.view.Surface import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer import io.github.thibaultbee.streampack.services.StreamerService @@ -53,12 +56,19 @@ open class SingleStreamerFactory( ) : StreamerFactory { override fun create(context: Context): ISingleStreamer { - return SingleStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + SingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withAudio) { + AudioOnlySingleStreamer(context) + } else { + VideoOnlySingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } } } diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 5cdbfe83b..8075db37f 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -163,8 +163,8 @@ class PreviewView @JvmOverloads constructor( streamer: IWithVideoSource ) { sourceJob += defaultScope.launch { - streamer.videoInput?.sourceFlow?.runningHistoryNotNull() - ?.collect { (previousVideoSource, newVideoSource) -> + streamer.videoInput.sourceFlow.runningHistoryNotNull() + .collect { (previousVideoSource, newVideoSource) -> if (previousVideoSource == newVideoSource) { Logger.w(TAG, "No change in video source") } else { @@ -448,7 +448,7 @@ class PreviewView @JvmOverloads constructor( private suspend fun stopPreview() { streamer?.let { - val videoSource = it.videoInput?.sourceFlow?.value + val videoSource = it.videoInput.sourceFlow.value if (videoSource is IPreviewableSource) { Logger.d(TAG, "Stopping preview") videoSource.previewMutex.withLock { diff --git a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt index 7134c77cd..68a4efd41 100644 --- a/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt @@ -18,7 +18,7 @@ suspend fun IWithVideoSource.setPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.setPreview(viewfinder, previewSize) } @@ -35,7 +35,7 @@ suspend fun IWithVideoSource.startPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.startPreview(viewfinder, previewSize) } From 31eaa48e3aad978d0a0a6a0a747d6754665d5fe7 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:09:41 +0100 Subject: [PATCH 26/27] feat(core): make DualStreamer inputs non nullable --- .../dual/file/CameraDualStreamerFileTest.kt | 1 + .../core/streamers/dual/DualStreamer.kt | 333 ++++++------------ .../core/streamers/dual/DualStreamerImpl.kt | 283 +++++++++++++++ .../core/streamers/dual/IDualStreamer.kt | 4 +- .../streamers/dual/VideoOnlyDualStreamer.kt | 72 +++- .../services/utils/StreamerFactory.kt | 20 +- 6 files changed, 457 insertions(+), 256 deletions(-) create mode 100644 core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index f8ce74d43..a148b8c03 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.dual.utils.DualStreamerCon import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.dual.cameraDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt index 72c92ffd0..e889d3aeb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt @@ -36,26 +36,15 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation -import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal -import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.combineTransform -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** * Creates a [DualStreamer] with a default audio source. @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraDualStreamer( @@ -74,19 +65,22 @@ suspend fun cameraDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( - context, - withAudio = true, - withVideo = true, - firstEndpointFactory, - secondEndpointFactory, - defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setCameraId(cameraId) if (audioSourceFactory != null) { - streamer.audioInput!!.setSource(audioSourceFactory) + streamer.setAudioSource(audioSourceFactory) } return streamer } @@ -99,6 +93,8 @@ suspend fun cameraDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionDualStreamer( @@ -106,18 +102,20 @@ suspend fun audioVideoMediaProjectionDualStreamer( mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - streamer.videoInput!!.setSource(MediaProjectionVideoSourceFactory(mediaProjection)) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) streamer.setAudioSource(MediaProjectionAudioSourceFactory(mediaProjection)) return streamer } @@ -131,6 +129,8 @@ suspend fun audioVideoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionDualStreamer( context: Context, @@ -138,16 +138,19 @@ suspend fun videoMediaProjectionDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) if (audioSourceFactory != null) { streamer.setAudioSource(audioSourceFactory) @@ -164,6 +167,8 @@ suspend fun videoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun DualStreamer( context: Context, @@ -171,16 +176,19 @@ suspend fun DualStreamer( videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, - withAudio = true, - withVideo = true, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) return streamer @@ -192,238 +200,93 @@ suspend fun DualStreamer( * For example, you can use it to live stream and record simultaneously. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class DualStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class DualStreamer( + context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), - dispatcherProvider: IDispatcherProvider = DispatcherProvider(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - AudioOutputMode.PUSH, - surfaceProcessorFactory, - dispatcherProvider - ) - - private val firstPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, firstEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - - /** - * First output of the streamer. - */ - override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - private val secondPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, secondEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - /** - * Second output of the streamer. - */ - override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - override val throwableFlow: StateFlow = merge( - pipeline.throwableFlow, - firstPipelineOutput.throwableFlow, - secondPipelineOutput.throwableFlow - ).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - /** - * Whether any of the output is opening. - */ - override val isOpenFlow: StateFlow = combineTransform( - firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow - ) { isOpens -> - emit(isOpens.any { it }) - }.stateIn( - coroutineScope, - SharingStarted.Eagerly, - false + private val streamer = DualStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - /** - * Whether any of the output is streaming. - */ - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow - - /** - * Closes the outputs. - * Same as calling [first.close] and [second.close]. - */ - override suspend fun close() { - firstPipelineOutput.close() - secondPipelineOutput.close() - } + override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput + override val second = streamer.second as IConfigurableVideoEncodingPipelineOutput - // SOURCES - override val audioInput = pipeline.audioInput + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // PROCESSORS - override val videoInput = pipeline.videoInput + override val audioInput: IAudioInput = streamer.audioInput + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Sets audio configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. - * - * @param audioConfig the audio configuration to set - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { - val throwables = mutableListOf() + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) = + streamer.setAudioConfig(audioConfig) - val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value - if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { - firstPipelineOutput.invalidateAudioCodecConfig() - } - val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value - if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( - audioConfig.secondAudioConfig - )) - ) { - secondPipelineOutput.invalidateAudioCodecConfig() - } + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) = + streamer.setVideoConfig(videoConfig) - try { - firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) - } catch (t: Throwable) { - throwables += t - } - try { - audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun close() = streamer.close() - /** - * Sets video configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. - * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or - * [second.setVideoCodecConfig] outputs. - * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, - * make sure that the frame rate for both configurations is the same. - * - * @param videoConfig the video configuration to set - */ - override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { - val throwables = mutableListOf() - - val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value - if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { - firstPipelineOutput.invalidateVideoCodecConfig() - } + override suspend fun startStream() = streamer.startStream() - val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value - if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( - videoConfig.secondVideoConfig - )) - ) { - secondPipelineOutput.invalidateVideoCodecConfig() - } - - try { - firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) - } catch (t: Throwable) { - throwables += t - } - try { - secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun stopStream() = streamer.stopStream() - /** - * Configures both video and audio settings. - * - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [DualStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig( - audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig - ) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } - - - override suspend fun startStream() = pipeline.startStream() - - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun release() = streamer.release() +} - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } +/** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun DualStreamer.setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig +) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } /** * Sets audio configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same audio configuration. * * @param audioConfig the audio configuration to set @@ -436,7 +299,7 @@ suspend fun DualStreamer.setAudioConfig(audioConfig: AudioConfig) { /** * Sets video configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same video configuration. * * @param videoConfig the video configuration to set diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt new file mode 100644 index 000000000..9455abd3f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt @@ -0,0 +1,283 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.dual + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.combineTransform +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + + +/** + * A class that handles 2 outputs. + * + * For example, you can use it to live stream and record simultaneously. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. + * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. + */ +internal class DualStreamerImpl( + private val context: Context, + withAudio: Boolean = true, + withVideo: Boolean = true, + firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { + private val coroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + AudioOutputMode.PUSH, + surfaceProcessorFactory, + dispatcherProvider + ) + + private val firstPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, firstEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + + /** + * First output of the streamer. + */ + override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + private val secondPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, secondEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + /** + * Second output of the streamer. + */ + override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + override val throwableFlow: StateFlow = merge( + pipeline.throwableFlow, + firstPipelineOutput.throwableFlow, + secondPipelineOutput.throwableFlow + ).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + /** + * Whether any of the output is opening. + */ + override val isOpenFlow: StateFlow = combineTransform( + firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow + ) { isOpens -> + emit(isOpens.any { it }) + }.stateIn( + coroutineScope, + SharingStarted.Eagerly, + false + ) + + /** + * Whether any of the output is streaming. + */ + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + /** + * Closes the outputs. + * Same as calling [first.close] and [second.close]. + */ + override suspend fun close() { + firstPipelineOutput.close() + secondPipelineOutput.close() + } + + // SOURCES + override val audioInput = pipeline.audioInput + + // PROCESSORS + override val videoInput = pipeline.videoInput + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Sets audio configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. + * + * @param audioConfig the audio configuration to set + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { + val throwables = mutableListOf() + + val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value + if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { + firstPipelineOutput.invalidateAudioCodecConfig() + } + val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value + if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( + audioConfig.secondAudioConfig + )) + ) { + secondPipelineOutput.invalidateAudioCodecConfig() + } + + try { + firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) + } catch (t: Throwable) { + throwables += t + } + try { + audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Sets video configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. + * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or + * [second.setVideoCodecConfig] outputs. + * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, + * make sure that the frame rate for both configurations is the same. + * + * @param videoConfig the video configuration to set + */ + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { + val throwables = mutableListOf() + + val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value + if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { + firstPipelineOutput.invalidateVideoCodecConfig() + } + + val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value + if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( + videoConfig.secondVideoConfig + )) + ) { + secondPipelineOutput.invalidateVideoCodecConfig() + } + + try { + firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) + } catch (t: Throwable) { + throwables += t + } + try { + secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + suspend fun setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig + ) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) + } + + + override suspend fun startStream() = pipeline.startStream() + + override suspend fun stopStream() = pipeline.stopStream() + + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt index 979ffe9a8..b98472b80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt @@ -243,9 +243,9 @@ internal constructor( val dynamicRangeProfile = firstVideoConfig.dynamicRangeProfile } -interface IAudioDualStreamer : IAudioStreamer +interface IAudioDualStreamer : IAudioStreamer, IDualStreamer -interface IVideoDualStreamer : IVideoStreamer +interface IVideoDualStreamer : IVideoStreamer, IDualStreamer interface IDualStreamer : ICloseableStreamer { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt index 31b89d58d..b4664f6b7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt @@ -23,14 +23,20 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig /** @@ -41,6 +47,8 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigu * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlyDualStreamer( @@ -48,10 +56,17 @@ suspend fun cameraVideoOnlyDualStreamer( cameraId: String = context.defaultCameraId, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( - context, firstEndpointFactory, secondEndpointFactory, defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer @@ -65,19 +80,25 @@ suspend fun cameraVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlyDualStreamer( context: Context, mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -91,19 +112,25 @@ suspend fun videoMediaProjectionVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlyDualStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -116,20 +143,26 @@ suspend fun VideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlyDualStreamer( context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IVideoDualStreamer { - private val streamer = DualStreamer( + private val streamer = DualStreamerImpl( context = context, - firstEndpointFactory = firstEndpointFactory, - secondEndpointFactory = secondEndpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput @@ -139,7 +172,7 @@ class VideoOnlyDualStreamer( override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. @@ -159,4 +192,17 @@ class VideoOnlyDualStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() -} \ No newline at end of file +} + +/** + * Sets video configuration. + * + * It is a shortcut for [VideoOnlyDualStreamer.setVideoConfig] when both + * outputs use the same video configuration. + * + * @param videoConfig the video configuration to set + */ +suspend fun VideoOnlyDualStreamer.setVideoConfig(videoConfig: VideoConfig) { + setVideoConfig(DualStreamerVideoConfig(videoConfig)) +} + diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 78e08fbab..d9866e2e4 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.VideoOnlyDualStreamer import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer @@ -86,11 +87,18 @@ open class DualStreamerFactory( ) : StreamerFactory { override fun create(context: Context): IDualStreamer { - return DualStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + DualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withVideo) { + VideoOnlyDualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else { + throw IllegalArgumentException("DualStreamer audio only is not supported yet") + } } } \ No newline at end of file From cc456b1586895df17c9be57876525fb87973c003 Mon Sep 17 00:00:00 2001 From: ThibaultBee <37510686+ThibaultBee@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:30:46 +0100 Subject: [PATCH 27/27] chore(version):bump to 3.2.0 snapshot --- build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index d1a8f90e6..0f5cb3ee9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,11 +5,11 @@ plugins { } allprojects { - val versionCode by extra { 3_001_000 } - val versionName by extra { "3.1.0" } + val versionCode by extra { 3_002_000 } + val versionName by extra { "3.2.0" } group = "io.github.thibaultbee.streampack" - version = versionName + version = "$versionName-SNAPSHOT" } dependencies {