diff --git a/build.gradle.kts b/build.gradle.kts index d1a8f90e6..0f5cb3ee9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,11 +5,11 @@ plugins { } allprojects { - val versionCode by extra { 3_001_000 } - val versionName by extra { "3.1.0" } + val versionCode by extra { 3_002_000 } + val versionName by extra { "3.2.0" } group = "io.github.thibaultbee.streampack" - version = versionName + version = "$versionName-SNAPSHOT" } dependencies { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt index f929a92b6..4405578e4 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt @@ -1,12 +1,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.content.Context -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubAudioSource : IAudioSourceInternal { private val _isStreamingFlow = MutableStateFlow(false) @@ -15,13 +14,11 @@ class StubAudioSource : IAudioSourceInternal { private val _configurationFlow = MutableStateFlow(null) val configurationFlow = _configurationFlow.asStateFlow() - override fun fillAudioFrame(frame: RawFrame): RawFrame { - return frame + override val minBufferSize = 0 + override fun fillAudioFrame(buffer: ByteBuffer): Long { + return 0L } - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0) - override suspend fun startStream() { _isStreamingFlow.value = true } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt index 22eca54d4..883c6ada1 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt @@ -8,12 +8,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.ISurfaceSour import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.VideoSourceConfig -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.time.Timebase import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME) : StubVideoSource(), @@ -44,8 +44,9 @@ class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME } class StubVideoFrameSource : StubVideoSource(), IVideoFrameSourceInternal { - override fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0L) + override fun getVideoFrame(buffer: ByteBuffer): Long { + return 0L + } class Factory : IVideoSourceInternal.Factory { override suspend fun create( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt index 8de63b080..d8591b2a3 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt @@ -83,8 +83,8 @@ class StreamerPipelineTest { ) streamerPipeline.addOutput(output) - val audioSource = streamerPipeline.audioInput?.sourceFlow?.value as IAudioSourceInternal - val videoSource = streamerPipeline.videoInput?.sourceFlow?.value as IVideoSourceInternal + val audioSource = streamerPipeline.audioInput.sourceFlow.value as IAudioSourceInternal + val videoSource = streamerPipeline.videoInput.sourceFlow.value as IVideoSourceInternal streamerPipeline.startStream() assertTrue(streamerPipeline.isStreamingFlow.first { it }) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 1531e0257..b2f1d4daf 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -20,7 +20,7 @@ import android.util.Log import androidx.core.net.toFile import androidx.test.platform.app.InstrumentationRegistry import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.DummyEndpoint @@ -317,7 +317,7 @@ class EncodingPipelineOutputTest { try { output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) @@ -330,14 +330,11 @@ class EncodingPipelineOutputTest { output.startStream(descriptor) output.queueAudioFrame( - RawFrame( + MutableRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) ) - - // Wait for frame to be received - assertNotNull(dummyEndpoint.frameFlow.filterNotNull().first()) } companion object { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index f8ce74d43..a148b8c03 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.dual.utils.DualStreamerCon import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.dual.cameraDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 548b4e053..607d4849e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.single.utils.SingleStreame import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt index 9a1f8a278..34da1422f 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt @@ -27,6 +27,7 @@ import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt index 0f5fcebd7..ee9232fe6 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt @@ -30,6 +30,7 @@ import io.github.thibaultbee.streampack.core.streamer.surface.SurfaceViewTestAct import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt index bfe9a2e05..81387d36b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import kotlinx.coroutines.test.runTest import org.junit.After diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt index 6571955c4..d4835c14e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt @@ -4,7 +4,7 @@ import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.Media import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer -import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext @@ -14,7 +14,7 @@ import kotlin.time.Duration.Companion.seconds object StreamerUtils { suspend fun runSingleStream( - streamer: SingleStreamer, + streamer: ISingleStreamer, descriptor: MediaDescriptor, duration: Duration, pollDuration: Duration = 1.seconds diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 018f4dbc1..ab615b1ad 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,29 +16,85 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.io.Closeable import java.nio.ByteBuffer /** - * A raw frame internal representation. + * Encoded frame representation */ -data class RawFrame( +interface RawFrame : Closeable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - var timestampInUs: Long, + val timestampInUs: Long +} + +/** + * Deep copy the [RawFrame.rawBuffer] into a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.deepCopy( + bufferPool: IBufferPool, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val copy = this.rawBuffer.deepCopy(bufferPool) + return copy( + rawBuffer = copy, timestampInUs = timestampInUs, onClosed = { + onClosed(it) + bufferPool.put(copy) + } + ) +} + +/** + * Copy a [RawFrame] to a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val pool = RawFramePool.default + return pool.get( + rawBuffer, timestampInUs, + { frame -> + onClosed(frame) + }) +} + +/** + * A mutable [RawFrame] internal representation. + * + * The purpose is to get reusable [RawFrame] + */ +data class MutableRawFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + /** + * Presentation timestamp in µs + */ + override var timestampInUs: Long, /** * A callback to call when frame is closed. */ - val onClosed: (RawFrame) -> Unit = {} -) : Closeable { + override var onClosed: (MutableRawFrame) -> Unit = {} +) : RawFrame, WithClosable { override fun close() { try { onClosed(this) @@ -48,119 +104,116 @@ data class RawFrame( } } - -data class Frame( +/** + * Encoded frame representation + */ +interface Frame : Closeable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - val ptsInUs: Long, + val ptsInUs: Long /** * Decoded timestamp in µs (not used). */ - val dtsInUs: Long? = null, + val dtsInUs: Long? /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - val isKeyFrame: Boolean, + val isKeyFrame: Boolean /** - * Contains csd buffers for key frames and audio frames only. + * Gets the csd buffers for key frames and audio frames. + * * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List?, + val extra: Extra? /** * Contains frame format.. * TODO: to remove */ val format: MediaFormat -) { - init { - removePrefixes() - } +} + +interface WithClosable { + val onClosed: (T) -> Unit } /** - * Removes the [Frame.extra] prefixes from the [Frame.rawBuffer]. - * - * With MediaCodec, the encoded frames may contain prefixes like SPS, PPS for H264/H265 key frames. - * It also modifies the position of the [Frame.rawBuffer] to skip the prefixes.s + * Copy a [Frame] to a new [Frame]. * - * @return A [ByteBuffer] without prefixes. + * For better memory allocation, you should close the returned frame after usage. */ -fun Frame.removePrefixes(): ByteBuffer { - return if (extra != null) { - rawBuffer.removePrefixes(extra) - } else { - rawBuffer - } +fun Frame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + ptsInUs: Long = this.ptsInUs, + dtsInUs: Long? = this.dtsInUs, + isKeyFrame: Boolean = this.isKeyFrame, + extra: Extra? = this.extra, + format: MediaFormat = this.format, + onClosed: (Frame) -> Unit = {} +): Frame { + val pool = FramePool.default + return pool.get( + rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, + { frame -> + onClosed(frame) + }) } -fun FrameWithCloseable( +/** + * A mutable [Frame] internal representation. + * + * The purpose is to get reusable [Frame] + */ +data class MutableFrame( /** * Contains an audio or video frame data. */ - rawBuffer: ByteBuffer, + override var rawBuffer: ByteBuffer, /** * Presentation timestamp in µs */ - ptsInUs: Long, + override var ptsInUs: Long, /** * Decoded timestamp in µs (not used). */ - dtsInUs: Long?, + override var dtsInUs: Long?, /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - isKeyFrame: Boolean, + override var isKeyFrame: Boolean, /** * Contains csd buffers for key frames and audio frames only. * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - extra: List?, + override var extra: Extra?, /** * Contains frame format.. + * TODO: to remove */ - format: MediaFormat, + override var format: MediaFormat, /** * A callback to call when frame is closed. */ - onClosed: (FrameWithCloseable) -> Unit, -) = FrameWithCloseable( - Frame( - rawBuffer, - ptsInUs, - dtsInUs, - isKeyFrame, - extra, - format - ), - onClosed -) - -/** - * Frame internal representation. - */ -data class FrameWithCloseable( - val frame: Frame, - val onClosed: (FrameWithCloseable) -> Unit -) : Closeable { + override var onClosed: (MutableFrame) -> Unit = {} +) : Frame, WithClosable { override fun close() { try { onClosed(this) @@ -171,8 +224,43 @@ data class FrameWithCloseable( } /** - * Uses the resource and unwraps the [Frame] to pass it to the given block. + * Ensures that extra are not used at the same time. + * + * After accessing the extra, they are automatically rewind for a new usage. + */ +class Extra(private val extraBuffers: List) { + private val lock = Any() + + val _length by lazy { extraBuffers.sumOf { it.remaining() } } + + fun getLength(): Int { + return synchronized(lock) { + _length + } + } + + fun get(extra: List.() -> T): T { + return synchronized(lock) { + val result = extraBuffers.extra() + extraBuffers.forEach { it.rewind() } + result + } + } +} + +/** + * Gets the duplicated extra. + * + * Prefers to use [Extra.get] as it does not create new resources. */ -inline fun FrameWithCloseable.useAndUnwrap(block: (Frame) -> T) = use { - block(frame) -} \ No newline at end of file +val Extra.extra: List + get() = get { this.map { it.duplicate() } } + +/** + * Gets the duplicated extra at [index] + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +fun Extra.get(index: Int): ByteBuffer { + return get { this[index].duplicate() } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt index 2bedd128a..b5d480a15 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt @@ -16,7 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.encoders import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendReleasable import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendStreamable @@ -77,7 +77,7 @@ interface IEncoderInternal : SuspendStreamable, SuspendReleasable, /** * A channel where the encoder will send encoded frames. */ - val outputChannel: SendChannel + val outputChannel: SendChannel } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index e62806ca5..9ebbde6a6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,7 +22,8 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Extra +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode import io.github.thibaultbee.streampack.core.elements.encoders.IEncoderInternal @@ -31,6 +32,8 @@ import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extens import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extensions.isValid import io.github.thibaultbee.streampack.core.elements.utils.extensions.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.put +import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -40,6 +43,8 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import java.io.Closeable +import java.nio.ByteBuffer import kotlin.math.min /** @@ -59,8 +64,8 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat - private var outputFormat: MediaFormat? = null - private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } + + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo, mediaCodec.outputFormat) } private val isVideo = encoderConfig.isVideo private val tag = if (isVideo) VIDEO_ENCODER_TAG else AUDIO_ENCODER_TAG + "(${this.hashCode()})" @@ -350,7 +355,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, outputFormat!!, info, tag + index, info, tag ) try { listener.outputChannel.send(frame) @@ -434,7 +439,6 @@ internal constructor( } override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - outputFormat = format Logger.i(tag, "Format changed : $format") } @@ -521,7 +525,7 @@ internal constructor( } val inputBuffer = requireNotNull(mediaCodec.getInputBuffer(inputBufferId)) val size = min(frame.rawBuffer.remaining(), inputBuffer.remaining()) - inputBuffer.put(frame.rawBuffer, frame.rawBuffer.position(), size) + inputBuffer.put(frame.rawBuffer, 0, size) mediaCodec.queueInputBuffer( inputBufferId, 0, size, frame.timestampInUs, 0 ) @@ -556,7 +560,6 @@ internal constructor( if (outputBufferId >= 0) { processOutputFrameUnsafe(mediaCodec, outputBufferId, bufferInfo) } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - outputFormat = mediaCodec.outputFormat Logger.i(tag, "Format changed: ${mediaCodec.outputFormat}") } } @@ -577,6 +580,18 @@ internal constructor( } } + fun FrameFactory( + codec: MediaCodec, + isVideo: Boolean, + outputFormat: MediaFormat + ): FrameFactory { + val extraBuffers: List? by lazy { outputFormat.extra } + val extra: Extra? by lazy { + extraBuffers?.map { it.duplicate() }?.let { Extra(it) } + } + return FrameFactory(codec, isVideo, outputFormat, extra, extraBuffers) + } + /** * A workaround to address the fact that some AAC encoders do not provide frame with `presentationTimeUs` in order. * If a frame is received with a timestamp lower or equal to the previous one, it is corrected by adding 1 to the previous timestamp. @@ -586,25 +601,32 @@ internal constructor( */ class FrameFactory( private val codec: MediaCodec, - private val isVideo: Boolean - ) { + private val isVideo: Boolean, + private val outputFormat: MediaFormat, + private val extra: Extra?, + private val extraBuffers: List? + ) : Closeable { private var previousPresentationTimeUs = 0L + private val pool = FramePool() + /** * Create a [Frame] from a [RawFrame] * * @return the created frame */ fun frame( - index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String - ): FrameWithCloseable { + index: Int, + info: BufferInfo, + tag: String + ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { pts = previousPresentationTimeUs + 1 Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, pts, info.isKeyFrame, tag) } /** @@ -617,26 +639,28 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, - outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, tag: String - ): FrameWithCloseable { + ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) - return FrameWithCloseable( - buffer, - ptsInUs, // pts - null, // dts + val extra = if (isKeyFrame || !isVideo) { + extra!! + } else { + null + } + val rawBuffer = if (extraBuffers != null) { + buffer.removePrefixes(extraBuffers) + } else { + buffer + } + + return pool.get( + rawBuffer, + ptsInUs, + null, isKeyFrame, - try { - if (isKeyFrame || !isVideo) { - outputFormat.extra - } else { - null - } - } catch (_: Throwable) { - null - }, + extra, outputFormat, onClosed = { try { @@ -647,6 +671,9 @@ internal constructor( }) } + override fun close() { + pool.close() + } } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 7d7d8f721..f9d235323 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -17,7 +17,8 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect import io.github.thibaultbee.streampack.core.logger.Logger @@ -36,7 +37,6 @@ import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch - /** * Combines multiple endpoints into one. * @@ -218,8 +218,7 @@ open class CombineEndpoint( * * If all endpoints write fails, it throws the exception of the first endpoint that failed. */ - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { val throwables = mutableListOf() /** @@ -231,9 +230,10 @@ open class CombineEndpoint( endpointInternals.filter { it.isOpenFlow.value }.forEach { endpoint -> try { val deferred = CompletableDeferred() - val duplicatedFrame = FrameWithCloseable( - frame.copy(rawBuffer = frame.rawBuffer.duplicate()), - { deferred.complete(Unit) }) + val duplicatedFrame = frame.copy( + rawBuffer = frame.rawBuffer.duplicate(), + onClosed = { deferred.complete(Unit) } + ) val endpointStreamId = endpointsToStreamIdsMap[Pair(endpoint, streamPid)]!! deferreds += deferred @@ -246,7 +246,7 @@ open class CombineEndpoint( coroutineScope.launch { deferreds.forEach { it.await() } - closeableFrame.close() + frame.close() } if (throwables.isNotEmpty()) { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt similarity index 80% rename from core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index b4a06c464..429b0a078 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -16,29 +16,24 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context -import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +/** + * A dummy endpoint for testing. + */ class DummyEndpoint : IEndpointInternal { private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() - private val _frameFlow = MutableStateFlow(null) - val frameFlow = _frameFlow.asStateFlow() - private val _isStreamingFlow = MutableStateFlow(false) val isStreamingFlow = _isStreamingFlow.asStateFlow() - private val _configFlow = MutableStateFlow(null) - val configFlow = _configFlow.asStateFlow() - override val info: IEndpoint.IEndpointInfo get() = TODO("Not yet implemented") @@ -58,20 +53,15 @@ class DummyEndpoint : IEndpointInternal { _isOpenFlow.emit(false) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame - Log.i(TAG, "write: $frame") - _frameFlow.emit(frame) - closeableFrame.close() + override suspend fun write(frame: Frame, streamPid: Int) { + frame.close() } override suspend fun addStreams(streamConfigs: List): Map { - streamConfigs.forEach { _configFlow.emit(it) } return streamConfigs.associateWith { it.hashCode() } } override suspend fun addStream(streamConfig: CodecConfig): Int { - _configFlow.emit(streamConfig) return streamConfig.hashCode() } @@ -82,10 +72,6 @@ class DummyEndpoint : IEndpointInternal { override suspend fun stopStream() { _isStreamingFlow.emit(false) } - - companion object { - private const val TAG = "DummyEndpoint" - } } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt index c242000ef..1951fabc0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.createDefaultTsServiceInfo -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer @@ -146,8 +146,8 @@ open class DynamicEndpoint( endpoint.addStream(streamConfig) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) = - safeEndpoint { endpoint -> endpoint.write(closeableFrame, streamPid) } + override suspend fun write(frame: Frame, streamPid: Int) = + safeEndpoint { endpoint -> endpoint.write(frame, streamPid) } override suspend fun startStream() = safeEndpoint { endpoint -> endpoint.startStream() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt index b87e8627f..715c7d5c8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.FileSink import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendCloseable @@ -46,14 +45,14 @@ interface IEndpointInternal : IEndpoint, SuspendStreamable, /** * Writes a [Frame] to the [IEndpointInternal]. * - * The [FrameWithCloseable.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. - * The [IEndpointInternal] must called [FrameWithCloseable.close] even if the frame is dropped or it somehow crashes. + * The [Frame.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. + * The [IEndpointInternal] must called [Frame.close] even if the frame is dropped or it somehow crashes. * - * @param closeableFrame the [Frame] to write + * @param frame the [Frame] to write * @param streamPid the stream id the [Frame] belongs to */ suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int, ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt index 1d30fba65..cedb2462a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt @@ -24,7 +24,7 @@ import android.media.MediaMuxer.OutputFormat import android.os.Build import android.os.ParcelFileDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider @@ -142,9 +142,8 @@ class MediaMuxerEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) = withContext(ioDispatcher) { - val frame = closeableFrame.frame mutex.withLock { try { if (state != State.STARTED && state != State.PENDING_START) { @@ -183,7 +182,7 @@ class MediaMuxerEndpoint( Logger.e(TAG, "Error while writing frame: ${t.message}") throw t } finally { - closeableFrame.close() + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt index 92f58fcee..e85726a65 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -80,9 +80,9 @@ class CompositeEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int - ) = muxer.write(closeableFrame, streamPid) + ) = muxer.write(frame, streamPid) override suspend fun addStreams(streamConfigs: List): Map { return mutex.withLock { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt index 1b609738b..108736b7c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt @@ -15,7 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable @@ -32,7 +32,7 @@ interface IMuxerInternal : fun onOutputFrame(packet: Packet) } - fun write(closeableFrame: FrameWithCloseable, streamPid: Int) + fun write(frame: Frame, streamPid: Int) fun addStreams(streamsConfig: List): Map diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt index 4c4f630e5..b96ca44b2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt @@ -15,9 +15,8 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4 -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet -import io.github.thibaultbee.streampack.core.elements.data.useAndUnwrap import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.FileTypeBox @@ -58,9 +57,9 @@ class Mp4Muxer( override val streamConfigs: List get() = tracks.map { it.config } - override fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { + override fun write(frame: Frame, streamPid: Int) { synchronized(this) { - closeableFrame.useAndUnwrap { frame -> + frame.use { frame -> if (segmenter!!.mustWriteSegment(frame)) { writeSegment() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt index 2ff9387c4..373b6eb42 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.unzip import java.nio.ByteBuffer @@ -50,8 +51,9 @@ class Chunk(val id: Int) { private val sampleSizes: List get() = samples.map { it.frame.rawBuffer.remaining() } + // TODO: pass `Extra` instead of `List` val extra: List> - get() = samples.mapNotNull { it.frame.extra }.unzip() + get() = samples.mapNotNull { it.frame.extra?.extra }.unzip() val format: List get() = samples.map { it.frame.format } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 076d1c132..28dde5d47 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import android.util.Size import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -55,7 +56,6 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.VPCodecConfigurationBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createHandlerBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createTypeMediaHeaderBox -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.AudioSpecificConfigDescriptor import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.ESDescriptor @@ -63,12 +63,13 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.SLCon import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodecConfigurationRecord -import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import java.nio.ByteBuffer /** @@ -175,7 +176,7 @@ class TrackChunks( } val frameCopy = - frame.copy(rawBuffer = frame.rawBuffer.clone()) // Do not keep mediacodec buffer + frame.copy(rawBuffer = frame.rawBuffer.deepCopy()) // Do not keep mediacodec buffer chunks.last().add(frameId, frameCopy) frameId++ } @@ -183,34 +184,38 @@ class TrackChunks( fun write() { chunks.forEach { chunk -> chunk.writeTo { frame -> - when (track.config.mimeType) { - MediaFormat.MIMETYPE_VIDEO_HEVC, - MediaFormat.MIMETYPE_VIDEO_AVC -> { - if (frame.rawBuffer.isAnnexB) { - // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() - val sizeBuffer = ByteBuffer.allocate(4) - sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) - onNewSample(sizeBuffer) - onNewSample(noStartCodeBuffer) - } else if (frame.rawBuffer.isAvcc) { - onNewSample(frame.rawBuffer) - } else { - throw IllegalArgumentException( - "Unsupported buffer format: buffer start with 0x${ - frame.rawBuffer.get( - 0 - ).toString(16) - }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ - frame.rawBuffer.get(2).toString(16) - }, 0x${frame.rawBuffer.get(3).toString(16)}" - ) + try { + when (track.config.mimeType) { + MediaFormat.MIMETYPE_VIDEO_HEVC, + MediaFormat.MIMETYPE_VIDEO_AVC -> { + if (frame.rawBuffer.isAnnexB) { + // Replace start code with size (from Annex B to AVCC) + val noStartCodeBuffer = frame.rawBuffer.skipStartCode() + val sizeBuffer = ByteBuffer.allocate(4) + sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) + onNewSample(sizeBuffer) + onNewSample(noStartCodeBuffer) + } else if (frame.rawBuffer.isAvcc) { + onNewSample(frame.rawBuffer) + } else { + throw IllegalArgumentException( + "Unsupported buffer format: buffer start with 0x${ + frame.rawBuffer.get( + 0 + ).toString(16) + }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ + frame.rawBuffer.get(2).toString(16) + }, 0x${frame.rawBuffer.get(3).toString(16)}" + ) + } } - } - else -> { - onNewSample(frame.rawBuffer) - } // Nothing + else -> { + onNewSample(frame.rawBuffer) + } // Nothing + } + } finally { + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index d0ab92de7..127662e43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal @@ -72,13 +72,12 @@ class TsMuxer : IMuxerInternal { /** * Encodes a frame to MPEG-TS format. * Each audio frames and each video key frames must come with an extra buffer containing sps, pps,... - * @param closeableFrame frame to mux + * @param frame frame to mux * @param streamPid Pid of frame stream. Throw a NoSuchElementException if streamPid refers to an unknown stream */ override fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame try { val pes = getPes(streamPid.toShort()) val mimeType = pes.stream.config.mimeType @@ -86,16 +85,17 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_AVC -> { // Copy sps & pps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for AVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -107,17 +107,18 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_HEVC -> { // Copy sps & pps & vps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for HEVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -133,11 +134,13 @@ class TsMuxer : IMuxerInternal { frame.rawBuffer, pes.stream.config as AudioCodecConfig ).toByteBuffer() } else { - LATMFrameWriter.fromDecoderSpecificInfo( - frame.rawBuffer, - frame.extra!!.first() - ) - .toByteBuffer() + frame.extra!!.get { + LATMFrameWriter.fromDecoderSpecificInfo( + frame.rawBuffer, + this.first() + ) + .toByteBuffer() + } } ) } @@ -157,11 +160,17 @@ class TsMuxer : IMuxerInternal { else -> throw IllegalArgumentException("Unsupported mimeType $mimeType") } - synchronized(this) { - generateStreams(newFrame, pes) + try { + synchronized(this) { + generateStreams(newFrame, pes) + } + } finally { + if (frame != newFrame) { + newFrame.close() + } } } finally { - closeableFrame.close() + frame.close() } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt new file mode 100644 index 000000000..0bad99623 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2025 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.processing + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IProcessor { + /** + * Process a data and returns a result. + * + * @param data data to process + * @return processed data + */ + fun process(data: T): T +} + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IEffectProcessor { + /** + * Process a data and returns a result. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + * @return processed data + */ + fun process(isMuted: Boolean, data: T): T +} + +/** + * Interface to process a data. + * + * @param T type of data to process + */ +interface IEffectConsumer { + /** + * Process a data. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + */ + fun consume(isMuted: Boolean, data: T) +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index 3dd308c16..aa5bfedd5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -16,8 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.processing import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFrameFactory +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -30,45 +31,40 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean -fun RawFramePullPush( - frameProcessor: IFrameProcessor, - onFrame: suspend (RawFrame) -> Unit, - processDispatcher: CoroutineDispatcher, - isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, RawFrameFactory(isDirect), processDispatcher) - /** * A component that pull a frame from an input and push it to [onFrame] output. * * @param frameProcessor the frame processor * @param onFrame the output frame callback - * @param frameFactory the frame factory to create frames + * @param bufferPool the [ByteBuffer] pool * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( - private val frameProcessor: IFrameProcessor, + private val frameProcessor: IProcessor, val onFrame: suspend (RawFrame) -> Unit, - private val frameFactory: IRawFrameFactory, + private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, ) { private val coroutineScope = CoroutineScope(SupervisorJob() + processDispatcher) private val mutex = Mutex() - private var getFrame: (suspend (frameFactory: IRawFrameFactory) -> RawFrame)? = null + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null private val isReleaseRequested = AtomicBoolean(false) private var job: Job? = null - suspend fun setInput(getFrame: suspend (frameFactory: IRawFrameFactory) -> RawFrame) { + suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } @@ -80,9 +76,11 @@ class RawFramePullPush( job = coroutineScope.launch { while (isActive) { val rawFrame = mutex.withLock { - val listener = getFrame ?: return@withLock null + val unwrapSource = source ?: return@withLock null try { - listener(frameFactory) + val buffer = bufferPool.get(unwrapSource.minBufferSize) + val timestampInUs = unwrapSource.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } catch (t: Throwable) { Logger.e(TAG, "Failed to get frame: ${t.message}") null @@ -94,7 +92,7 @@ class RawFramePullPush( // Process buffer with effects val processedFrame = try { - frameProcessor.processFrame(rawFrame) + frameProcessor.process(rawFrame) } catch (t: Throwable) { Logger.e(TAG, "Failed to pre-process frame: ${t.message}") continue @@ -115,7 +113,8 @@ class RawFramePullPush( job?.cancel() job = null - frameFactory.clear() + pool.clear() + bufferPool.clear() } fun release() { @@ -127,7 +126,8 @@ class RawFramePullPush( } coroutineScope.cancel() - frameFactory.close() + pool.close() + bufferPool.close() } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt index 83af9a5a7..bde293f80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt @@ -16,30 +16,47 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IEffectConsumer +import io.github.thibaultbee.streampack.core.elements.processing.IEffectProcessor import java.io.Closeable /** * The base audio effect. */ -interface IAudioEffect : IFrameProcessor, Closeable +sealed interface IAudioEffect : Closeable /** - * Mute audio effect. + * An audio effect that can be dispatched to another thread. The result is not use by the audio pipeline. + * Example: a VU meter. */ -class MuteEffect : IAudioEffect { +interface IConsumerAudioEffect : IAudioEffect, IEffectConsumer + +/** + * An audio effect that can't be dispatched to another thread. The result is used by the audio pipeline. + * + * The [RawFrame.rawBuffer] can't be modified. + */ +interface IProcessorAudioEffect : IAudioEffect, IEffectProcessor + +/** + * An audio effect that mute the audio. + */ +class MuteEffect : IProcessorAudioEffect { private var mutedByteArray: ByteArray? = null - override fun processFrame(frame: RawFrame): RawFrame { - val remaining = frame.rawBuffer.remaining() - val position = frame.rawBuffer.position() + override fun process(isMuted: Boolean, data: RawFrame): RawFrame { + if (!isMuted) { + return data + } + val remaining = data.rawBuffer.remaining() + val position = data.rawBuffer.position() if (remaining != mutedByteArray?.size) { mutedByteArray = ByteArray(remaining) } - frame.rawBuffer.put(mutedByteArray!!) - frame.rawBuffer.position(position) + data.rawBuffer.put(mutedByteArray!!) + data.rawBuffer.position(position) - return frame + return data } override fun close() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 4cd6e7b7a..f10f6af43 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,22 +16,76 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.data.deepCopy +import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import java.io.Closeable +import java.nio.ByteBuffer +import java.util.concurrent.CopyOnWriteArrayList +import java.util.function.IntFunction /** * Audio frame processor. - * - * Only supports mute effect for now. */ -class AudioFrameProcessor : IFrameProcessor, - IAudioFrameProcessor { +class AudioFrameProcessor( + private val bufferPool: IBufferPool, + dispatcher: CoroutineDispatcher, + private val effects: CopyOnWriteArrayList = CopyOnWriteArrayList() +) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { + private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) + + /** + * Whether the audio is muted. + * + * When the audio is muted, the audio effect are not processed. Only consumer effects are processed. + */ override var isMuted = false private val muteEffect = MuteEffect() - override fun processFrame(frame: RawFrame): RawFrame { - if (isMuted) { - return muteEffect.processFrame(frame) + private fun launchConsumerEffect( + effect: IConsumerAudioEffect, + isMuted: Boolean, + data: RawFrame + ) { + val consumeFrame = data.deepCopy(bufferPool) + coroutineScope.launch { + effect.consume(isMuted, consumeFrame) + } + } + + override fun process(data: RawFrame): RawFrame { + val isMuted = isMuted + + var processedFrame = muteEffect.process(isMuted, data) + + effects.forEach { + if (it is IProcessorAudioEffect) { + processedFrame = it.process(isMuted, processedFrame) + } else if (it is IConsumerAudioEffect) { + launchConsumerEffect(it, isMuted, processedFrame) + } } - return frame + + return processedFrame + } + + override fun close() { + effects.forEach { it.close() } + effects.clear() + + muteEffect.close() + + coroutineScope.cancel() + } + + @Deprecated("'fun toArray(generator: IntFunction!>!): Array<(out) T!>!' is deprecated. This declaration is redundant in Kotlin and might be removed soon.") + @Suppress("DEPRECATION") + override fun toArray(generator: IntFunction?>): Array { + return super.toArray(generator) } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt index 24c7606e1..19cc6ba9c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt @@ -20,7 +20,7 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio */ interface IAudioFrameProcessor { /** - * Mute audio. + * Whether the processor is muted. */ var isMuted: Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt index 18c89bb6c..209ae9ea8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt @@ -15,26 +15,20 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.audio -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IAudioFrameSourceInternal { /** - * Gets an audio frame from the source. - * - * @param frame the [RawFrame] to fill with audio data. - * @return a [RawFrame] containing audio data. + * Gets the size of the buffer to allocate. + * When using encoder is callback mode, it's unused. */ - fun fillAudioFrame(frame: RawFrame): RawFrame + val minBufferSize: Int /** * Gets an audio frame from the source. * - * The [RawFrame] to fill with audio data is created by the [frameFactory]. - * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing audio data. + * @param buffer the [ByteBuffer] to fill with audio data. + * @return the timestamp in microseconds. */ - fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun fillAudioFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt index 324438aaf..cc1746614 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt @@ -22,18 +22,18 @@ import android.media.AudioTimestamp import android.media.audiofx.AudioEffect import android.os.Build import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Companion.isValidUUID import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.getFactoryForEffectType +import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.availableEffect import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.isEffectAvailable -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.extensions.type -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer import java.util.UUID /** @@ -42,7 +42,9 @@ import java.util.UUID */ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSource { private var audioRecord: AudioRecord? = null - private var bufferSize: Int? = null + private var _minBufferSize: Int? = null + override val minBufferSize: Int + get() = _minBufferSize ?: throw IllegalStateException("Audio source is not initialized") private var processor: EffectProcessor? = null private var pendingAudioEffects = mutableListOf() @@ -74,9 +76,9 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } } - bufferSize = getMinBufferSize(config) + _minBufferSize = getMinBufferSize(config) - audioRecord = buildAudioRecord(config, bufferSize!!).also { + audioRecord = buildAudioRecord(config, _minBufferSize!!).also { val previousEffects = processor?.getAll() ?: emptyList() processor?.clear() @@ -157,30 +159,19 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } - override fun fillAudioFrame(frame: RawFrame): RawFrame { + override fun fillAudioFrame( + buffer: ByteBuffer + ): Long { val audioRecord = requireNotNull(audioRecord) { "Audio source is not initialized" } if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { throw IllegalStateException("Audio source is not recording") } - val buffer = frame.rawBuffer val length = audioRecord.read(buffer, buffer.remaining()) - if (length > 0) { - frame.timestampInUs = getTimestampInUs(audioRecord) - return frame - } else { - frame.close() + if (length <= 0) { throw IllegalArgumentException(audioRecordErrorToString(length)) } - } - - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame { - val bufferSize = requireNotNull(bufferSize) { "Buffer size is not initialized" } - - /** - * Dummy timestamp: it is overwritten later. - */ - return fillAudioFrame(frameFactory.create(bufferSize, 0)) + return getTimestampInUs(audioRecord) } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt index 3c206900d..a90d61103 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt @@ -15,17 +15,15 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IVideoFrameSourceInternal { /** * Gets a video frame from a source. * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing video data. + * @param buffer The buffer to fill with the video frame. + * @return The timestamp in microseconds of the video frame. */ - fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun getVideoFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index bc87cc182..91fc4f7b9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -20,8 +20,13 @@ import android.graphics.PointF import android.graphics.Rect import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_ACTIVE_SCAN +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_FOCUSED_LOCKED +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_INACTIVE +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.hardware.camera2.params.ColorSpaceTransform import android.hardware.camera2.params.MeteringRectangle import android.hardware.camera2.params.RggbChannelVector @@ -48,6 +53,7 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.scalerMaxZoom import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.sensitivityRange import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.zoomRatioRange +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.extensions.clamp import io.github.thibaultbee.streampack.core.elements.utils.extensions.isApplicationPortrait import io.github.thibaultbee.streampack.core.elements.utils.extensions.isNormalized @@ -55,10 +61,20 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.launchIn import io.github.thibaultbee.streampack.core.elements.utils.extensions.normalize import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.CancellationException +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicLong /** @@ -77,6 +93,29 @@ class CameraSettings internal constructor( */ val cameraId = cameraController.cameraId + @RequiresApi(Build.VERSION_CODES.Q) + private fun getPhysicalCameraIdCallbackFlow() = callbackFlow { + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + trySend(result.get(CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID)!!) + return false + } + } + cameraController.addCaptureCallbackListener(captureCallback) + awaitClose { + runBlocking { + cameraController.removeCaptureCallbackListener(captureCallback) + } + } + }.conflate().distinctUntilChanged() + + /** + * Current physical camera id. + */ + val physicalCameraIdFlow: Flow + @RequiresApi(Build.VERSION_CODES.Q) + get() = getPhysicalCameraIdCallbackFlow() + /** * Whether the camera is available. * To be used before calling any camera settings. @@ -152,15 +191,87 @@ class CameraSettings internal constructor( * Applies settings to the camera repeatedly in a synchronized way. * * This method returns when the capture callback is received with the passed request. + * + * @return the total capture result */ - suspend fun applyRepeatingSessionSync() = cameraController.setRepeatingSessionSync() + suspend fun applyRepeatingSessionSync(): TotalCaptureResult { + val deferred = CompletableDeferred() + + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + deferred.complete(result) + return false + } + } + applyRepeatingSession(captureResult) + return deferred.await() + } + + /** + * Applies settings to the camera repeatedly. + * + * @param onCaptureResult the capture result callback. Return `true` to stop the callback. + */ + suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { + val tag = TagBundle.Factory.default.create() + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + val resultTag = result.request.tag as? TagBundle + val keyId = resultTag?.keyId ?: return false + if (keyId >= tag.keyId) { + return onCaptureResult.onCaptureResult(result) + } + return false + } + } + + cameraController.addCaptureCallbackListener(captureCallback) + cameraController.setRepeatingSession(tag) + } /** * Applies settings to the camera repeatedly. */ suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() - class Flash( + private class TagBundle(val keyId: Long) { + private val tagMap = mutableMapOf().apply { + put(TAG_KEY_ID, keyId) + } + + val keys: Set + get() = tagMap.keys + + fun setTag(key: String, value: Any?) { + tagMap[key] = value + } + + companion object { + private const val TAG_KEY_ID = "TAG_KEY_ID" + } + + /** + * Factory for [TagBundle]. + * + * The purpose is to make sure the tag always contains an increasing id. + */ + class Factory private constructor() { + /** + * Next session id. + */ + private val nextSessionUpdateId = AtomicLong(0) + + fun create(): TagBundle { + return TagBundle(nextSessionUpdateId.getAndIncrement()) + } + + companion object { + val default = Factory() + } + } + } + + class Flash internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -169,8 +280,7 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, [Boolean.Companion.toString] otherwise. */ - val isAvailable: Boolean - get() = characteristics.isFlashAvailable + val isAvailable: Boolean by lazy { characteristics.isFlashAvailable } /** * Enables or disables flash. @@ -210,9 +320,44 @@ class CameraSettings internal constructor( cameraSettings.set(CaptureRequest.FLASH_MODE, mode) cameraSettings.applyRepeatingSession() } + + /** + * Gets the range of supported flash strength. + * Range is from [1-x]. + * + * Use the range to call [setStrengthLevel] + */ + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val availableStrengthLevelRange: Range by lazy { + Range( + 1, + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] ?: 1 + ) + } + + val strengthLevel: Int + /** + * Gets the flash strength. + * + * @return the flash strength + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) ?: 1 + + /** + * Sets the flash strength. + * + * @param level flash strength. Range is from [1-x]. + * @see [availableStrengthLevelRange] + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + suspend fun setStrengthLevel(level: Int) { + cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) + cameraSettings.applyRepeatingSession() + } } - class WhiteBalance( + class WhiteBalance internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -221,8 +366,7 @@ class CameraSettings internal constructor( * * @return list of supported white balance modes. */ - val availableAutoModes: List - get() = characteristics.autoWhiteBalanceModes + val availableAutoModes: List by lazy { characteristics.autoWhiteBalanceModes } /** * Gets the auto white balance mode. @@ -253,8 +397,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available white balance metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfWhiteBalanceMeteringRegions + val maxNumOfMeteringRegions: Int by lazy { characteristics.maxNumberOfWhiteBalanceMeteringRegions } /** * Gets the white balance metering regions. @@ -290,7 +433,7 @@ class CameraSettings internal constructor( } } - class Iso( + class Iso internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -301,8 +444,9 @@ class CameraSettings internal constructor( * * @see [sensorSensitivity] */ - val availableSensorSensitivityRange: Range - get() = characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + val availableSensorSensitivityRange: Range by lazy { + characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + } /** * Gets lens focus distance. @@ -338,7 +482,7 @@ class CameraSettings internal constructor( } } - class ColorCorrection( + class ColorCorrection internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -347,11 +491,10 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, `false` otherwise. */ - val isAvailable: Boolean - get() { - return characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] - ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true - } + val isAvailable: Boolean by lazy { + characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] + ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true + } /** * Gets color correction gain. @@ -393,7 +536,7 @@ class CameraSettings internal constructor( } } - class Exposure( + class Exposure internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -404,9 +547,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoExposureModes - + val availableAutoModes: List by lazy { characteristics.autoExposureModes } /** * Gets auto exposure mode. @@ -440,9 +581,10 @@ class CameraSettings internal constructor( * @see [availableCompensationStep] * @see [compensation] */ - val availableCompensationRange: Range - get() = characteristics.exposureRange + val availableCompensationRange: Range by lazy { + characteristics.exposureRange ?: DEFAULT_COMPENSATION_RANGE + } /** * Gets current camera exposure compensation step. @@ -456,9 +598,9 @@ class CameraSettings internal constructor( * @see [availableCompensationRange] * @see [compensation] */ - val availableCompensationStep: Rational - get() = characteristics.exposureStep - ?: DEFAULT_COMPENSATION_STEP_RATIONAL + val availableCompensationStep: Rational by lazy { + characteristics.exposureStep ?: DEFAULT_COMPENSATION_STEP_RATIONAL + } /** * Gets exposure compensation. @@ -491,8 +633,7 @@ class CameraSettings internal constructor( /** * Get maximum number of available exposure metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfExposureMeteringRegions + val maxNumOfMeteringRegions by lazy { characteristics.maxNumberOfExposureMeteringRegions } /** * Gets the exposure metering regions. @@ -544,6 +685,8 @@ class CameraSettings internal constructor( protected val characteristics: CameraCharacteristics, protected val cameraSettings: CameraSettings ) { + private val listeners: CopyOnWriteArrayList = CopyOnWriteArrayList() + abstract val availableRatioRange: Range internal abstract suspend fun getCropSensorRegion(): Rect @@ -569,7 +712,21 @@ class CameraSettings internal constructor( } } - class CropScalerRegionZoom( + fun addListener(listener: OnZoomChangedListener) { + listeners.add(listener) + } + + fun removeListener(listener: OnZoomChangedListener) { + listeners.remove(listener) + } + + protected fun notifyZoomListeners(zoomRatio: Float) { + listeners.forEach { + it.onZoomChanged(zoomRatio) + } + } + + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { @@ -579,10 +736,11 @@ class CameraSettings internal constructor( private var persistentZoomRatio = 1f private var currentCropRect: Rect? = null - override val availableRatioRange: Range - get() = Range( + override val availableRatioRange: Range by lazy { + Range( DEFAULT_ZOOM_RATIO, characteristics.scalerMaxZoom ) + } override suspend fun getZoomRatio(): Float = mutex.withLock { persistentZoomRatio @@ -591,6 +749,11 @@ class CameraSettings internal constructor( override suspend fun setZoomRatio(zoomRatio: Float) { mutex.withLock { val clampedValue = zoomRatio.clamp(availableRatioRange) + if (clampedValue == persistentZoomRatio) { + return@withLock + } + persistentZoomRatio = clampedValue + currentCropRect = getCropRegion( characteristics, clampedValue @@ -598,8 +761,8 @@ class CameraSettings internal constructor( cameraSettings.set( CaptureRequest.SCALER_CROP_REGION, currentCropRect ) - cameraSettings.applyRepeatingSession() - persistentZoomRatio = clampedValue + cameraSettings.applyRepeatingSessionSync() + notifyZoomListeners(clampedValue) } } @@ -644,11 +807,14 @@ class CameraSettings internal constructor( } @RequiresApi(Build.VERSION_CODES.R) - class RZoom(characteristics: CameraCharacteristics, cameraSettings: CameraSettings) : + class RZoom internal constructor( + characteristics: CameraCharacteristics, + cameraSettings: CameraSettings + ) : Zoom(characteristics, cameraSettings) { - override val availableRatioRange: Range - get() = characteristics.zoomRatioRange - ?: DEFAULT_ZOOM_RATIO_RANGE + override val availableRatioRange: Range by lazy { + characteristics.zoomRatioRange ?: DEFAULT_ZOOM_RATIO_RANGE + } override suspend fun getZoomRatio(): Float { return cameraSettings.get(CaptureRequest.CONTROL_ZOOM_RATIO) @@ -656,10 +822,14 @@ class CameraSettings internal constructor( } override suspend fun setZoomRatio(zoomRatio: Float) { + if (zoomRatio == getZoomRatio()) { + return + } cameraSettings.set( CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio.clamp(availableRatioRange) ) cameraSettings.applyRepeatingSession() + notifyZoomListeners(zoomRatio) } override suspend fun getCropSensorRegion(): Rect { @@ -671,7 +841,7 @@ class CameraSettings internal constructor( const val DEFAULT_ZOOM_RATIO = 1f val DEFAULT_ZOOM_RATIO_RANGE = Range(DEFAULT_ZOOM_RATIO, DEFAULT_ZOOM_RATIO) - fun build( + internal fun build( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ): Zoom { @@ -682,10 +852,22 @@ class CameraSettings internal constructor( } } } + + /** + * Listener for zoom change. + */ + interface OnZoomChangedListener { + /** + * Called when the zoom ratio changes. + * + * @param zoomRatio the zoom ratio + */ + fun onZoomChanged(zoomRatio: Float) + } } - class Focus( + class Focus internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -696,8 +878,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoFocusModes + val availableAutoModes: List by lazy { characteristics.autoFocusModes } /** * Gets the auto focus mode. @@ -730,8 +911,9 @@ class CameraSettings internal constructor( * * @see [lensDistance] */ - val availableLensDistanceRange: Range - get() = characteristics.lensDistanceRange + val availableLensDistanceRange: Range by lazy { + characteristics.lensDistanceRange + } /** * Gets the lens focus distance. @@ -764,9 +946,9 @@ class CameraSettings internal constructor( /** * Get maximum number of available focus metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfFocusMeteringRegions - ?: DEFAULT_MAX_NUM_OF_METERING_REGION + val maxNumOfMeteringRegions: Int by lazy { + characteristics.maxNumberOfFocusMeteringRegions ?: DEFAULT_MAX_NUM_OF_METERING_REGION + } /** * Gets the focus metering regions. @@ -795,7 +977,7 @@ class CameraSettings internal constructor( } } - class Stabilization( + class Stabilization internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -839,8 +1021,9 @@ class CameraSettings internal constructor( * * @see [isEnableOptical] */ - val isOpticalAvailable: Boolean - get() = characteristics.isOpticalStabilizationAvailable + val isOpticalAvailable: Boolean by lazy { + characteristics.isOpticalStabilizationAvailable + } /** @@ -879,7 +1062,7 @@ class CameraSettings internal constructor( } } - class FocusMetering( + class FocusMetering internal constructor( private val coroutineScope: CoroutineScope, private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings, @@ -951,7 +1134,35 @@ class CameraSettings internal constructor( ) cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, aeMode) } - cameraSettings.applyRepeatingSession() + + val deferred = CompletableDeferred() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + return when (val afState = result[CaptureResult.CONTROL_AF_STATE]) { + CONTROL_AF_STATE_FOCUSED_LOCKED -> { + deferred.complete(Unit) + true + } + + CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> { + deferred.completeExceptionally(Exception("AF not focused")) + true + } + + CONTROL_AF_STATE_INACTIVE -> { + deferred.completeExceptionally(CancellationException("AF has been cancelled")) + true + } + + CONTROL_AF_STATE_ACTIVE_SCAN -> false + else -> { + deferred.completeExceptionally(IllegalStateException("AF is not in an expected state $afState")) + } + } + } + } + cameraSettings.applyRepeatingSession(captureResult) + deferred.await() } private suspend fun executeMetering( @@ -968,7 +1179,7 @@ class CameraSettings internal constructor( */ cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, - CameraMetadata.CONTROL_AF_TRIGGER_IDLE + CameraMetadata.CONTROL_AF_TRIGGER_CANCEL ) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { cameraSettings.set( @@ -979,9 +1190,6 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSessionSync() addFocusMetering(afRectangles, aeRectangles, awbRectangles) - if (afRectangles.isNotEmpty()) { - triggerAf(true) - } // Auto cancel AF trigger after timeoutDurationMs if (timeoutDurationMs > 0) { @@ -994,6 +1202,10 @@ class CameraSettings internal constructor( } } } + + if (afRectangles.isNotEmpty()) { + triggerAf(true) + } } private suspend fun startFocusAndMetering( diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 13c33e653..a2fca02b0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -23,9 +23,11 @@ import android.util.Range import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineScope @@ -57,6 +59,8 @@ internal class CameraController( private var deviceController: CameraDeviceController? = null private var sessionController: CameraSessionController? = null + private val sessionCallback = CameraSessionCallback(coroutineScope) + private val controllerMutex = Mutex() private val outputs = mutableMapOf() @@ -160,8 +164,9 @@ internal class CameraController( val deviceController = getDeviceController() CameraSessionController.create( coroutineScope, - sessionCompat, deviceController, + sessionCallback, + sessionCompat, outputs.values.toList(), dynamicRange = dynamicRangeProfile.dynamicRange, fpsRange = fpsRange, @@ -378,21 +383,31 @@ internal class CameraController( } /** - * Sets a repeating session sync with the current capture request. + * Adds a capture callback listener to the current capture session. * - * It returns only when the capture callback has been called for the first time. + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ - suspend fun setRepeatingSessionSync() { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSessionSync() + suspend fun addCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.addListener(listener) + } + + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + suspend fun removeCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.removeListener(listener) } /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { + suspend fun setRepeatingSession(tag: Any? = null) { val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSession() + sessionController.setRepeatingSession(tag) } private suspend fun closeControllers() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index fc3843e3f..b42274eca 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -23,6 +23,7 @@ import android.hardware.camera2.TotalCaptureResult import android.util.Range import android.view.Surface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder @@ -33,19 +34,16 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.atomic.AtomicLong -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine internal class CameraSessionController private constructor( private val coroutineScope: CoroutineScope, + private val captureSession: CameraCaptureSession, private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, + private val sessionCallback: CameraSessionCallback, private val sessionCompat: ICameraCaptureSessionCompat, - private val captureSession: CameraCaptureSession, private val outputs: List, val dynamicRange: Long, val cameraIsClosedFlow: StateFlow, @@ -58,8 +56,6 @@ internal class CameraSessionController private constructor( private val requestTargetMutex = Mutex() - private val nextSessionUpdateId = AtomicLong(0) - /** * A default capture callback that logs the failure reason. */ @@ -72,10 +68,8 @@ internal class CameraSessionController private constructor( } } - private val sessionCallback = CameraControlSessionCallback(coroutineScope) - private val captureCallbacks = - mutableSetOf(captureCallback, sessionCallback) + setOf(captureCallback, sessionCallback) suspend fun isEmpty() = withContext(coroutineScope.coroutineContext) { requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } @@ -250,25 +244,12 @@ internal class CameraSessionController private constructor( } } - suspend fun removeCaptureCallback( - cameraCaptureCallback: CaptureCallback - ) { - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - captureCallbacks.remove(cameraCaptureCallback) - } - } - } - /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSessionSync() { + suspend fun setRepeatingSession(tag: Any? = null) { if (captureRequestBuilder.isEmpty()) { Logger.w(TAG, "Capture request is empty") return @@ -280,52 +261,7 @@ internal class CameraSessionController private constructor( return@withContext } - suspendCoroutine { continuation -> - val nextSessionId = nextSessionUpdateId.getAndIncrement() - - val captureCallback = object : CaptureResultListener { - override fun onCaptureResult(result: TotalCaptureResult): Boolean { - val tag = result.request.tag as? TagBundle - val keyId = tag?.keyId ?: return false - if (keyId >= nextSessionId) { - continuation.resume(Unit) - return true - } - return false - } - } - - sessionCallback.addListener(captureCallback) - - captureRequestBuilder.setTag( - TagBundle().apply { - keyId = nextSessionId - } - ) - sessionCompat.setRepeatingSingleRequest( - captureSession, - captureRequestBuilder.build(), - MultiCaptureCallback(captureCallbacks) - ) - } - } - } - } - - /** - * Sets a repeating session with the current capture request. - */ - suspend fun setRepeatingSession() { - if (captureRequestBuilder.isEmpty()) { - Logger.w(TAG, "Capture request is empty") - return - } - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } + tag?.let { captureRequestBuilder.setTag(it) } sessionCompat.setRepeatingSingleRequest( captureSession, @@ -414,9 +350,10 @@ internal class CameraSessionController private constructor( val controller = CameraSessionController( coroutineScope, + newCaptureSession, captureRequestBuilder, + sessionCallback, sessionCompat, - newCaptureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, @@ -436,8 +373,9 @@ internal class CameraSessionController private constructor( suspend fun create( coroutineScope: CoroutineScope, - sessionCompat: ICameraCaptureSessionCompat, cameraDeviceController: CameraDeviceController, + sessionCallback: CameraSessionCallback, + sessionCompat: ICameraCaptureSessionCompat, outputs: List, dynamicRange: Long, fpsRange: Range, @@ -465,9 +403,10 @@ internal class CameraSessionController private constructor( } return CameraSessionController( coroutineScope, + captureSession, captureRequestBuilder, + sessionCallback, sessionCompat, - captureSession, outputs, dynamicRange, cameraDeviceController.isClosedFlow, @@ -513,68 +452,4 @@ internal class CameraSessionController private constructor( } } } - - private class TagBundle { - private val tagMap = mutableMapOf() - - var keyId: Long? - get() = tagMap[TAG_KEY_ID] as? Long - set(value) { - tagMap[TAG_KEY_ID] = value - } - - val keys: Set - get() = tagMap.keys - - fun setTag(key: String, value: Any?) { - tagMap[key] = value - } - - companion object { - private const val TAG_KEY_ID = "TAG_KEY_ID" - } - } - - interface CaptureResultListener { - /** - * Called when a capture result is received. - * - * @param result The capture result. - * @return true if the listener is finished and should be removed, false otherwise. - */ - fun onCaptureResult(result: TotalCaptureResult): Boolean - } - - private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : - CaptureCallback() { - /* synthetic accessor */ - private val resultListeners = mutableSetOf() - - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) - } - - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) - } - - override fun onCaptureCompleted( - session: CameraCaptureSession, - request: CaptureRequest, - result: TotalCaptureResult - ) { - coroutineScope.launch { - val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) - } - } - if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) - } - } - } - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt new file mode 100644 index 000000000..67b8b086d --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCaptureSession.CaptureCallback +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.TotalCaptureResult +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ +internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) : + CaptureCallback() { + /* synthetic accessor */ + private val resultListeners = mutableSetOf() + private val mutex = Mutex() + + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ + suspend fun addListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.add(listener) + } + } + + suspend fun removeListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.remove(listener) + } + } + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + coroutineScope.launch { + val removeSet = mutableSetOf() + mutex.withLock { + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } + } + } + if (!removeSet.isEmpty()) { + mutex.withLock { + resultListeners.removeAll(removeSet) + } + } + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt similarity index 50% rename from core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt index dc9375392..c97eeda2f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt @@ -1,11 +1,11 @@ /* - * Copyright 2025 Thibault B. + * Copyright (C) 2026 Thibault B. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.github.thibaultbee.streampack.core.elements.processing +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils -import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import android.hardware.camera2.TotalCaptureResult -/** - * Interface to process a frame. - * - * @param T type of frame to process (probably [RawFrame] or [Frame]) - */ -interface IFrameProcessor { - fun processFrame(frame: T): T +interface CaptureResultListener { + /** + * Called when a capture result is received. + * + * @param result The capture result. + * @return true if the listener is finished and should be removed, false otherwise. + */ + fun onCaptureResult(result: TotalCaptureResult): Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt index 75281c053..0b0c0fd3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt @@ -19,8 +19,8 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.buffer.ByteBuffer import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaFormat import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import java.nio.ByteBuffer @@ -33,8 +33,8 @@ data class AVCDecoderConfigurationRecord( private val sps: List, private val pps: List ) : ByteBufferWriter() { - private val spsNoStartCode: List = sps.map { it.removeStartCode() } - private val ppsNoStartCode: List = pps.map { it.removeStartCode() } + private val spsNoStartCode: List = sps.map { it.skipStartCode() } + private val ppsNoStartCode: List = pps.map { it.skipStartCode() } override val size: Int = getSize(spsNoStartCode, ppsNoStartCode) @@ -86,11 +86,13 @@ data class AVCDecoderConfigurationRecord( sps: List, pps: List ): AVCDecoderConfigurationRecord { - val spsNoStartCode = sps.map { it.removeStartCode() } - val ppsNoStartCode = pps.map { it.removeStartCode() } - val profileIdc: Byte = spsNoStartCode[0].get(1) - val profileCompatibility = spsNoStartCode[0].get(2) - val levelIdc = spsNoStartCode[0].get(3) + val spsNoStartCode = sps.map { it.skipStartCode() } + val ppsNoStartCode = pps.map { it.skipStartCode() } + val firstSpsNoStartCode = spsNoStartCode[0] + val firstSpsNoStartCodePosition = firstSpsNoStartCode.position() + val profileIdc: Byte = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 1) + val profileCompatibility = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 2) + val levelIdc = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 3) return AVCDecoderConfigurationRecord( profileIdc = profileIdc, profileCompatibility = profileCompatibility, @@ -112,7 +114,8 @@ data class AVCDecoderConfigurationRecord( size += 2 + it.remaining() - it.startCodeSize } val spsStartCodeSize = sps[0].startCodeSize - val profileIdc = sps[0].get(spsStartCodeSize + 1).toInt() + val spsPosition = sps[0].position() + val profileIdc = sps[0].get(spsPosition + spsStartCodeSize + 1).toInt() if ((profileIdc == 100) || (profileIdc == 110) || (profileIdc == 122) || (profileIdc == 144)) { size += 4 } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt index 28a11669f..7bd86d908 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt @@ -20,7 +20,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaForma import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putLong48 import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl import io.github.thibaultbee.streampack.core.elements.utils.extensions.shr import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize @@ -195,7 +195,7 @@ data class HEVCDecoderConfigurationRecord( } data class NalUnit(val type: Type, val data: ByteBuffer, val completeness: Boolean = true) { - val noStartCodeData: ByteBuffer = data.removeStartCode() + val noStartCodeData: ByteBuffer = data.skipStartCode() fun write(buffer: ByteBuffer) { buffer.put((completeness shl 7) or type.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 66da15b18..09cee1ff1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.extensions +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.charset.StandardCharsets @@ -86,12 +87,14 @@ fun ByteBuffer.put3x3Matrix(matrix: IntArray) { matrix.forEach { putInt(it) } } -fun ByteBuffer.put(buffer: ByteBuffer, offset: Int, length: Int) { - val limit = buffer.limit() - buffer.position(offset) - buffer.limit(offset + length) - this.put(buffer) - buffer.limit(limit) +fun ByteBuffer.put(src: ByteBuffer, offset: Int, length: Int) { + val limit = src.limit() + if (offset != 0) { + src.position(src.position() + offset) + } + src.limit(src.position() + offset + length) + this.put(src) + src.limit(limit) } fun ByteBuffer.getString(size: Int = this.remaining()): String { @@ -111,22 +114,38 @@ fun ByteBuffer.getLong(isLittleEndian: Boolean): Long { return value } -fun ByteBuffer.indicesOf(prefix: ByteArray): List { - if (prefix.isEmpty()) { - return emptyList() +/** + * Finds all occurrences of the given [needle] byte array within the ByteBuffer. + * @param needle The byte array sequence to search for. + * @return A list of starting indices for every match found. + */ +fun ByteBuffer.indicesOf(needle: ByteArray): List { + if (needle.isEmpty()) return emptyList() + + val results = mutableListOf() + val end = limit() - needle.size + var i = position() + + while (i <= end) { + if (match(i, needle)) { + results.add(i) + // Move forward by the needle's length to find the next non-overlapping match + // Or use i++ if you want to allow overlapping matches (e.g., "AAA" in "AAAA") + i += needle.size + } else { + i++ + } } + return results +} - val indices = mutableListOf() - - outer@ for (i in 0 until this.limit() - prefix.size + 1) { - for (j in prefix.indices) { - if (this.get(i + j) != prefix[j]) { - continue@outer - } +private fun ByteBuffer.match(start: Int, needle: ByteArray): Boolean { + for (idx in needle.indices) { + if (get(start + idx) != needle[idx]) { + return false } - indices.add(i) } - return indices + return true } /** @@ -137,18 +156,18 @@ fun ByteBuffer.slices(prefix: ByteArray): List { // Get all occurrence of prefix in buffer val indexes = this.indicesOf(prefix) + // Get slices indexes.forEachIndexed { index, i -> val nextPosition = if (indexes.indices.contains(index + 1)) { - indexes[index + 1] - 1 + indexes[index + 1] } else { - this.limit() - 1 + this.limit() } slices.add(Pair(i, nextPosition)) } - val array = this.array() return slices.map { - ByteBuffer.wrap(array.sliceArray(IntRange(it.first, it.second))) + this.slice(from = it.first, to = it.second) } } @@ -235,18 +254,37 @@ fun ByteBuffer.toByteArray(): ByteArray { } /** - * Clone [ByteBuffer]. + * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ -fun ByteBuffer.clone(): ByteBuffer { +@Deprecated("Use ByteBufferPool instead") +fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { - val clone = if (isDirect) { + val copy = if (isDirect) { ByteBuffer.allocateDirect(this.remaining()) } else { ByteBuffer.allocate(this.remaining()) } - return clone.put(this).apply { rewind() } + return copy.put(this).apply { rewind() } + } finally { + this.position(originalPosition) + } +} + +/** + * Deep copy of [ByteBuffer] from [IBufferPool]. + * + * Don't forget to put the returned [ByteBuffer] to the buffer pool when you are done with it. + * + * @param pool [IBufferPool] to use + * @return [ByteBuffer] deep copy + */ +fun ByteBuffer.deepCopy(pool: IBufferPool): ByteBuffer { + val originalPosition = this.position() + try { + val copy = pool.get(this.remaining()) + return copy.put(this).apply { rewind() } } finally { this.position(originalPosition) } @@ -258,51 +296,72 @@ fun ByteBuffer.clone(): ByteBuffer { /** - * Get start code size of [ByteBuffer]. + * Get start code size of [ByteBuffer] from the current position */ val ByteBuffer.startCodeSize: Int - get() { - return if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x00.toByte() && this.get(3) == 0x01.toByte() - ) { - 4 - } else if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x01.toByte() - ) { - 3 - } else { - 0 - } + get() = getStartCodeSize(this.position()) + +/** + * Get start code size of [ByteBuffer] from the given [position]. + */ +fun ByteBuffer.getStartCodeSize( + position: Int +): Int { + return if (remaining() >= 4 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x00.toByte() && this.get(position + 3) == 0x01.toByte() + ) { + 4 + } else if (remaining() >= 3 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x01.toByte() + ) { + 3 + } else { + 0 } +} -fun ByteBuffer.removeStartCode(): ByteBuffer { +/** + * Moves the position after the start code. + */ +fun ByteBuffer.skipStartCode(): ByteBuffer { val startCodeSize = this.startCodeSize - this.position(startCodeSize) - return this.slice() + if (startCodeSize > 0) { + this.position(this.position() + startCodeSize) + } + return this } +private val emulationPreventionThreeByte = byteArrayOf(0x00, 0x00, 0x03) + +/** + * Removes all emulation prevention three bytes from [ByteBuffer]. + * + * @param headerLength [Int] of the header length before writing the [ByteBuffer]. + * @return [ByteBuffer] without emulation prevention three bytes + */ fun ByteBuffer.extractRbsp(headerLength: Int): ByteBuffer { - val rbsp = ByteBuffer.allocateDirect(this.remaining()) + val indices = this.indicesOf(emulationPreventionThreeByte) - val indices = this.indicesOf(byteArrayOf(0x00, 0x00, 0x03)) + val rbspSize = + this.remaining() - indices.size // remove 0x3 bytes for each emulation prevention three bytes + val rbsp = ByteBuffer.allocateDirect(rbspSize) - rbsp.put(this, this.startCodeSize, headerLength) + // Write header to new buffer + rbsp.put(this, 0, headerLength + this.startCodeSize) - var previous = this.position() indices.forEach { - rbsp.put(this, previous, it + 2 - previous) - previous = it + 3 // skip emulation_prevention_three_byte + rbsp.put(this, 0, it + 2 - this.position()) + this.position(this.position() + 1) // skip emulation_prevention_three_byte } - rbsp.put(this, previous, this.limit() - previous) + rbsp.put(this, 0, this.limit() - this.position()) - rbsp.limit(rbsp.position()) rbsp.rewind() return rbsp } /** * Remove all [prefixes] from [ByteBuffer] whatever their order. - * It slices [ByteBuffer] so it does not copy data. + * It moves the [position] of the [ByteBuffer]. * * Once a prefix is found, it is removed from the [prefixes] list. * @@ -324,7 +383,7 @@ fun ByteBuffer.removePrefixes(prefixes: List): ByteBuffer { } } - return this.slice().order(this.order()) + return this } /** @@ -343,4 +402,21 @@ val ByteBuffer.isAvcc: Boolean get() { val size = this.getInt(0) return size == (this.remaining() - 4) - } \ No newline at end of file + } + +/** + * Slices the buffer from [from] position to [to] position. + * + * @param from start position + * @param to end position + */ +fun ByteBuffer.slice(from: Int, to: Int): ByteBuffer { + val currentPosition = this.position() + val currentLimit = this.limit() + this.position(from) + this.limit(to) + val newBuffer = this.slice() + this.position(currentPosition) + this.limit(currentLimit) + return newBuffer +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt index ce748def0..f3036b9b8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt @@ -87,4 +87,4 @@ class ByteBufferPool(private val isDirect: Boolean) : IBufferPool, C buffers.clear() } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt new file mode 100644 index 000000000..c1e824e97 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame +import java.nio.ByteBuffer + + +/** + * A pool of [MutableFrame]. + */ +internal class FramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + ptsInUs: Long, + dtsInUs: Long?, + isKeyFrame: Boolean, + extra: Extra?, + format: MediaFormat, + onClosed: (MutableFrame) -> Unit + ): MutableFrame { + val frame = get() + + val onClosedHook = { frame: MutableFrame -> + onClosed(frame) + put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.ptsInUs = ptsInUs + frame.dtsInUs = dtsInUs + frame.isKeyFrame = isKeyFrame + frame.extra = extra + frame.format = format + frame.onClosed = onClosedHook + frame + } else { + MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format, + onClosed = onClosedHook + ) + } + } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { FramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt deleted file mode 100644 index 759419483..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt +++ /dev/null @@ -1,31 +0,0 @@ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -interface IReadOnlyRawFrameFactory { - /** - * Creates a [RawFrame]. - * - * The returned frame must be released by calling [RawFrame.close] when it is not used anymore. - * - * @param bufferSize the buffer size - * @param timestampInUs the frame timestamp in µs - * @return a frame - */ - fun create(bufferSize: Int, timestampInUs: Long): RawFrame -} - -/** - * A pool of frames. - */ -interface IRawFrameFactory : IReadOnlyRawFrameFactory { - /** - * Clears the factory. - */ - fun clear() - - /** - * Closes the factory. - */ - fun close() -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt new file mode 100644 index 000000000..73bc0b337 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPool.kt @@ -0,0 +1,90 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import io.github.thibaultbee.streampack.core.logger.Logger +import java.io.Closeable +import java.util.ArrayDeque +import java.util.concurrent.atomic.AtomicBoolean + +/** + * A pool of objects. + * + * The pool is thread-safe. + * + * The implementation is required to add a `get` methods. + * + * @param T the type of object to pool + */ +internal sealed class ObjectPool() : Closeable { + private val pool = ArrayDeque() + + private val isClosed = AtomicBoolean(false) + + protected fun get(): T? { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + + return synchronized(pool) { + if (!pool.isEmpty()) { + pool.removeFirst() + } else { + null + } + } + } + + /** + * Puts an object in the pool. + * + * @param any the object to put + */ + fun put(any: T) { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + synchronized(pool) { + pool.addLast(any) + } + } + + /** + * Clears the pool. + */ + fun clear() { + if (isClosed.get()) { + return + } + synchronized(pool) { + pool.clear() + } + } + + /** + * Closes the pool. + * + * After a pool is closed, it cannot be used anymore. + */ + override fun close() { + if (isClosed.getAndSet(true)) { + return + } + synchronized(pool) { + pool.clear() + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt deleted file mode 100644 index dc4b3032b..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.logger.Logger - -/** - * A factory to create [RawFrame]. - */ -fun RawFrameFactory(isDirect: Boolean): RawFrameFactory { - return RawFrameFactory(ByteBufferPool(isDirect)) -} - -/** - * A factory to create [RawFrame]. - */ -class RawFrameFactory(private val bufferPool: ByteBufferPool) : IRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - try { - bufferPool.put(rawFrame.rawBuffer) - } catch (t: Throwable) { - Logger.w(TAG, "Error while putting buffer in pool: $t") - } - } - } - - override fun clear() { - bufferPool.clear() - } - - override fun close() { - bufferPool.close() - } - - companion object { - private const val TAG = "RawFramePool" - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt new file mode 100644 index 000000000..0c7b4e6a0 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import io.github.thibaultbee.streampack.core.elements.data.MutableRawFrame +import java.nio.ByteBuffer + + +/** + * A pool of [MutableRawFrame]. + */ +internal class RawFramePool() : ObjectPool() { + fun get( + rawBuffer: ByteBuffer, + timestampInUs: Long, + onClosed: (MutableRawFrame) -> Unit = {} + ): MutableRawFrame { + val frame = get() + + val onClosedHook = { frame: MutableRawFrame -> + onClosed(frame) + put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.timestampInUs = timestampInUs + frame.onClosed = onClosedHook + frame + } else { + MutableRawFrame( + rawBuffer = rawBuffer, + timestampInUs = timestampInUs, + onClosed = onClosedHook + ) + } + } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { RawFramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt index 60a476805..80ae9b9e4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt @@ -36,7 +36,7 @@ interface IWithAudioSource { /** * The audio input to access to advanced settings. */ - val audioInput: IAudioInput? + val audioInput: IAudioInput /** * Sets a new audio source. @@ -44,8 +44,7 @@ interface IWithAudioSource { * @param audioSourceFactory The new audio source factory. */ suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) { - val audio = requireNotNull(audioInput) { "Audio input is not available" } - audio.setSource(audioSourceFactory) + audioInput.setSource(audioSourceFactory) } } @@ -63,9 +62,9 @@ interface IWithVideoRotation { */ interface IWithVideoSource { /** - * The audio input to access to advanced settings. + * The video input to access to advanced settings. */ - val videoInput: IVideoInput? + val videoInput: IVideoInput /** * Sets the video source. @@ -73,8 +72,7 @@ interface IWithVideoSource { * The previous video source will be released unless its preview is still running. */ suspend fun setVideoSource(videoSourceFactory: IVideoSourceInternal.Factory) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(videoSourceFactory) + videoInput.setSource(videoSourceFactory) } } @@ -87,8 +85,7 @@ interface IWithVideoSource { */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setCameraId(cameraId: String) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(CameraSourceFactory(cameraId)) + videoInput.setSource(CameraSourceFactory(cameraId)) } @@ -96,7 +93,7 @@ suspend fun IWithVideoSource.setCameraId(cameraId: String) { * Whether the video source has a preview. */ val IWithVideoSource.isPreviewable: Boolean - get() = videoInput?.sourceFlow?.value is IPreviewableSource + get() = videoInput.sourceFlow?.value is IPreviewableSource /** * Sets the preview surface. @@ -105,7 +102,7 @@ val IWithVideoSource.isPreviewable: Boolean * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.setPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.setPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.setPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -142,7 +139,7 @@ suspend fun IWithVideoSource.setPreview(textureView: TextureView) = * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.startPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview() ?: throw IllegalStateException("Video source is not previewable") } @@ -154,7 +151,7 @@ suspend fun IWithVideoSource.startPreview() { * @see [IWithVideoSource.stopPreview] */ suspend fun IWithVideoSource.startPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -195,5 +192,5 @@ suspend fun IWithVideoSource.startPreview(textureView: TextureView) = * Stops video preview. */ suspend fun IWithVideoSource.stopPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.stopPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.stopPreview() } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index bb745019c..28e53f39e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.pipelines import android.content.Context import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory @@ -70,7 +71,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.Closeable import java.util.concurrent.atomic.AtomicBoolean /** @@ -116,7 +116,8 @@ open class StreamerPipeline( } else { null } - override val audioInput: IAudioInput? = _audioInput + override val audioInput: IAudioInput + get() = requireNotNull(_audioInput) { "Audio input is not available" } private val _videoInput = if (withVideo) { VideoInput(context, surfaceProcessorFactory, dispatcherProvider) { @@ -126,7 +127,8 @@ open class StreamerPipeline( null } - override val videoInput: IVideoInput? = _videoInput + override val videoInput: IVideoInput + get() = requireNotNull(_videoInput) { "Video input is not available" } private val _isStreamingFlow = MutableStateFlow(false) @@ -246,25 +248,17 @@ open class StreamerPipeline( Logger.e(TAG, "Error while queueing audio frame to output: $t") } } else { - // Hook to close frame when all outputs have processed it - var numOfClosed = 0 - val onClosed = { frame: Closeable -> - numOfClosed++ - if (numOfClosed == audioStreamingOutput.size) { - frame.close() - } - } audioStreamingOutput.forEachIndexed { index, output -> try { output.queueAudioFrame( - frame.copy( - rawBuffer = if (index == audioStreamingOutput.lastIndex) { - frame.rawBuffer - } else { - frame.rawBuffer.duplicate() - }, - onClosed = onClosed - ) + if (index == audioStreamingOutput.lastIndex) { + frame + } else { + frame.copy( + rawBuffer = + frame.rawBuffer.duplicate() + ) + } ) } catch (t: Throwable) { Logger.e(TAG, "Error while queueing audio frame to output $output: $t") diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 05812e4d8..999bc16e8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -24,10 +24,12 @@ import io.github.thibaultbee.streampack.core.elements.processing.RawFramePullPus import io.github.thibaultbee.streampack.core.elements.processing.audio.AudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.processing.audio.IAudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING import io.github.thibaultbee.streampack.core.pipelines.IAudioDispatcherProvider @@ -131,15 +133,17 @@ internal class AudioInput( } // PROCESSOR + private val bufferPool = ByteBufferPool(true) + /** * The audio processor. */ - private val frameProcessorInternal = AudioFrameProcessor() - override val processor: IAudioFrameProcessor = frameProcessorInternal + private val processorInternal = AudioFrameProcessor(bufferPool, dispatcherProvider.default) + override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(frameProcessorInternal, config, dispatcherProvider) + PushAudioPort(processorInternal, config, bufferPool, dispatcherProvider) } else { - CallbackAudioPort(frameProcessorInternal) // No threading needed, called from encoder thread + CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } // CONFIG @@ -200,15 +204,7 @@ internal class AudioInput( } } - when (port) { - is PushAudioPort -> { - port.setInput(newAudioSource::getAudioFrame) - } - - is CallbackAudioPort -> { - port.setInput(newAudioSource::fillAudioFrame) - } - } + port.setInput(newAudioSource) // Replace audio source sourceInternalFlow.emit(newAudioSource) @@ -362,6 +358,15 @@ internal class AudioInput( ) } + try { + processorInternal.close() + } catch (t: Throwable) { + Logger.w( + TAG, + "release: Can't close audio processor: ${t.message}" + ) + } + isStreamingJob.cancel() } coroutineScope.coroutineContext.cancelChildren() @@ -380,27 +385,29 @@ internal class AudioInput( internal class CallbackConfig : Config() } -private sealed interface IAudioPort : Streamable, Releasable { - suspend fun setInput(getFrame: T) +private sealed interface IAudioPort : Streamable, Releasable { + suspend fun setInput(source: IAudioFrameSourceInternal) suspend fun removeInput() } private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, + bufferPool: ByteBufferPool, dispatcherProvider: IAudioDispatcherProvider -) : IAudioPort<(frameFactory: IRawFrameFactory) -> RawFrame> { +) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, + bufferPool, dispatcherProvider.createAudioDispatcher( 1, THREAD_NAME_AUDIO_PREPROCESSING ) ) - override suspend fun setInput(getFrame: (frameFactory: IRawFrameFactory) -> RawFrame) { - audioPullPush.setInput(getFrame) + override suspend fun setInput(source: IAudioFrameSourceInternal) { + audioPullPush.setInput(source) } override suspend fun removeInput() { @@ -421,32 +428,35 @@ private class PushAudioPort( } private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProcessor) : - IAudioPort<(frame: RawFrame) -> RawFrame> { - private var getFrame: ((frame: RawFrame) -> RawFrame)? = null + IAudioPort { private val mutex = Mutex() + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null var audioFrameRequestedListener: OnFrameRequestedListener = object : OnFrameRequestedListener { override suspend fun onFrameRequested(buffer: ByteBuffer): RawFrame { val frame = mutex.withLock { - val getFrame = requireNotNull(getFrame) { + val source = requireNotNull(source) { "Audio frame requested listener is not set yet" } - getFrame(RawFrame(buffer, 0)) + val timestampInUs = source.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } - return audioFrameProcessor.processFrame(frame) + return audioFrameProcessor.process(frame) } } - override suspend fun setInput(getFrame: (frame: RawFrame) -> RawFrame) { + override suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } override suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index cedbad83b..d3b8a2a05 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.pipelines.outputs.encoding import android.content.Context import android.view.Surface import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -239,7 +239,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -250,7 +250,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -259,10 +259,10 @@ internal class EncodingPipelineOutput( if (withAudio) { coroutineScope.launch(audioOutputDispatcher) { // Audio - audioEncoderListener.outputChannel.consumeEach { closeableFrame -> + audioEncoderListener.outputChannel.consumeEach { frame -> try { audioStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Audio frame received but audio stream is not set") } catch (t: Throwable) { onInternalError(t) @@ -273,10 +273,10 @@ internal class EncodingPipelineOutput( if (withVideo) { coroutineScope.launch(videoOutputDispatcher) { // Video - videoEncoderListener.outputChannel.consumeEach { closeableFrame -> + videoEncoderListener.outputChannel.consumeEach { frame -> try { videoStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Video frame received but video stream is not set") } catch (t: Throwable) { onInternalError(t) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt index 72c92ffd0..e889d3aeb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt @@ -36,26 +36,15 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation -import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal -import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.combineTransform -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** * Creates a [DualStreamer] with a default audio source. @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraDualStreamer( @@ -74,19 +65,22 @@ suspend fun cameraDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( - context, - withAudio = true, - withVideo = true, - firstEndpointFactory, - secondEndpointFactory, - defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setCameraId(cameraId) if (audioSourceFactory != null) { - streamer.audioInput!!.setSource(audioSourceFactory) + streamer.setAudioSource(audioSourceFactory) } return streamer } @@ -99,6 +93,8 @@ suspend fun cameraDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionDualStreamer( @@ -106,18 +102,20 @@ suspend fun audioVideoMediaProjectionDualStreamer( mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - streamer.videoInput!!.setSource(MediaProjectionVideoSourceFactory(mediaProjection)) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) streamer.setAudioSource(MediaProjectionAudioSourceFactory(mediaProjection)) return streamer } @@ -131,6 +129,8 @@ suspend fun audioVideoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionDualStreamer( context: Context, @@ -138,16 +138,19 @@ suspend fun videoMediaProjectionDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) if (audioSourceFactory != null) { streamer.setAudioSource(audioSourceFactory) @@ -164,6 +167,8 @@ suspend fun videoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun DualStreamer( context: Context, @@ -171,16 +176,19 @@ suspend fun DualStreamer( videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, - withAudio = true, - withVideo = true, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) return streamer @@ -192,238 +200,93 @@ suspend fun DualStreamer( * For example, you can use it to live stream and record simultaneously. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class DualStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class DualStreamer( + context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), - dispatcherProvider: IDispatcherProvider = DispatcherProvider(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - AudioOutputMode.PUSH, - surfaceProcessorFactory, - dispatcherProvider - ) - - private val firstPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, firstEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - - /** - * First output of the streamer. - */ - override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - private val secondPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, secondEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - /** - * Second output of the streamer. - */ - override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - override val throwableFlow: StateFlow = merge( - pipeline.throwableFlow, - firstPipelineOutput.throwableFlow, - secondPipelineOutput.throwableFlow - ).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - /** - * Whether any of the output is opening. - */ - override val isOpenFlow: StateFlow = combineTransform( - firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow - ) { isOpens -> - emit(isOpens.any { it }) - }.stateIn( - coroutineScope, - SharingStarted.Eagerly, - false + private val streamer = DualStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - /** - * Whether any of the output is streaming. - */ - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow - - /** - * Closes the outputs. - * Same as calling [first.close] and [second.close]. - */ - override suspend fun close() { - firstPipelineOutput.close() - secondPipelineOutput.close() - } + override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput + override val second = streamer.second as IConfigurableVideoEncodingPipelineOutput - // SOURCES - override val audioInput = pipeline.audioInput + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // PROCESSORS - override val videoInput = pipeline.videoInput + override val audioInput: IAudioInput = streamer.audioInput + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Sets audio configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. - * - * @param audioConfig the audio configuration to set - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { - val throwables = mutableListOf() + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) = + streamer.setAudioConfig(audioConfig) - val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value - if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { - firstPipelineOutput.invalidateAudioCodecConfig() - } - val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value - if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( - audioConfig.secondAudioConfig - )) - ) { - secondPipelineOutput.invalidateAudioCodecConfig() - } + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) = + streamer.setVideoConfig(videoConfig) - try { - firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) - } catch (t: Throwable) { - throwables += t - } - try { - audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun close() = streamer.close() - /** - * Sets video configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. - * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or - * [second.setVideoCodecConfig] outputs. - * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, - * make sure that the frame rate for both configurations is the same. - * - * @param videoConfig the video configuration to set - */ - override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { - val throwables = mutableListOf() - - val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value - if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { - firstPipelineOutput.invalidateVideoCodecConfig() - } + override suspend fun startStream() = streamer.startStream() - val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value - if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( - videoConfig.secondVideoConfig - )) - ) { - secondPipelineOutput.invalidateVideoCodecConfig() - } - - try { - firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) - } catch (t: Throwable) { - throwables += t - } - try { - secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun stopStream() = streamer.stopStream() - /** - * Configures both video and audio settings. - * - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [DualStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig( - audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig - ) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } - - - override suspend fun startStream() = pipeline.startStream() - - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun release() = streamer.release() +} - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } +/** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun DualStreamer.setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig +) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } /** * Sets audio configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same audio configuration. * * @param audioConfig the audio configuration to set @@ -436,7 +299,7 @@ suspend fun DualStreamer.setAudioConfig(audioConfig: AudioConfig) { /** * Sets video configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same video configuration. * * @param videoConfig the video configuration to set diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt new file mode 100644 index 000000000..9455abd3f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt @@ -0,0 +1,283 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.dual + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.combineTransform +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + + +/** + * A class that handles 2 outputs. + * + * For example, you can use it to live stream and record simultaneously. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. + * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. + */ +internal class DualStreamerImpl( + private val context: Context, + withAudio: Boolean = true, + withVideo: Boolean = true, + firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { + private val coroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + AudioOutputMode.PUSH, + surfaceProcessorFactory, + dispatcherProvider + ) + + private val firstPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, firstEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + + /** + * First output of the streamer. + */ + override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + private val secondPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, secondEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + /** + * Second output of the streamer. + */ + override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + override val throwableFlow: StateFlow = merge( + pipeline.throwableFlow, + firstPipelineOutput.throwableFlow, + secondPipelineOutput.throwableFlow + ).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + /** + * Whether any of the output is opening. + */ + override val isOpenFlow: StateFlow = combineTransform( + firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow + ) { isOpens -> + emit(isOpens.any { it }) + }.stateIn( + coroutineScope, + SharingStarted.Eagerly, + false + ) + + /** + * Whether any of the output is streaming. + */ + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + /** + * Closes the outputs. + * Same as calling [first.close] and [second.close]. + */ + override suspend fun close() { + firstPipelineOutput.close() + secondPipelineOutput.close() + } + + // SOURCES + override val audioInput = pipeline.audioInput + + // PROCESSORS + override val videoInput = pipeline.videoInput + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Sets audio configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. + * + * @param audioConfig the audio configuration to set + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { + val throwables = mutableListOf() + + val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value + if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { + firstPipelineOutput.invalidateAudioCodecConfig() + } + val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value + if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( + audioConfig.secondAudioConfig + )) + ) { + secondPipelineOutput.invalidateAudioCodecConfig() + } + + try { + firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) + } catch (t: Throwable) { + throwables += t + } + try { + audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Sets video configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. + * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or + * [second.setVideoCodecConfig] outputs. + * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, + * make sure that the frame rate for both configurations is the same. + * + * @param videoConfig the video configuration to set + */ + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { + val throwables = mutableListOf() + + val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value + if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { + firstPipelineOutput.invalidateVideoCodecConfig() + } + + val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value + if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( + videoConfig.secondVideoConfig + )) + ) { + secondPipelineOutput.invalidateVideoCodecConfig() + } + + try { + firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) + } catch (t: Throwable) { + throwables += t + } + try { + secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + suspend fun setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig + ) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) + } + + + override suspend fun startStream() = pipeline.startStream() + + override suspend fun stopStream() = pipeline.stopStream() + + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt index 979ffe9a8..b98472b80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt @@ -243,9 +243,9 @@ internal constructor( val dynamicRangeProfile = firstVideoConfig.dynamicRangeProfile } -interface IAudioDualStreamer : IAudioStreamer +interface IAudioDualStreamer : IAudioStreamer, IDualStreamer -interface IVideoDualStreamer : IVideoStreamer +interface IVideoDualStreamer : IVideoStreamer, IDualStreamer interface IDualStreamer : ICloseableStreamer { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt index 31b89d58d..b4664f6b7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt @@ -23,14 +23,20 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig /** @@ -41,6 +47,8 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigu * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlyDualStreamer( @@ -48,10 +56,17 @@ suspend fun cameraVideoOnlyDualStreamer( cameraId: String = context.defaultCameraId, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( - context, firstEndpointFactory, secondEndpointFactory, defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer @@ -65,19 +80,25 @@ suspend fun cameraVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlyDualStreamer( context: Context, mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -91,19 +112,25 @@ suspend fun videoMediaProjectionVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlyDualStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -116,20 +143,26 @@ suspend fun VideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlyDualStreamer( context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IVideoDualStreamer { - private val streamer = DualStreamer( + private val streamer = DualStreamerImpl( context = context, - firstEndpointFactory = firstEndpointFactory, - secondEndpointFactory = secondEndpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput @@ -139,7 +172,7 @@ class VideoOnlyDualStreamer( override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. @@ -159,4 +192,17 @@ class VideoOnlyDualStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() -} \ No newline at end of file +} + +/** + * Sets video configuration. + * + * It is a shortcut for [VideoOnlyDualStreamer.setVideoConfig] when both + * outputs use the same video configuration. + * + * @param videoConfig the video configuration to set + */ +suspend fun VideoOnlyDualStreamer.setVideoConfig(videoConfig: VideoConfig) { + setVideoConfig(DualStreamerVideoConfig(videoConfig)) +} + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt index d8043700c..2408dda37 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt @@ -25,8 +25,9 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput -import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo /** @@ -35,15 +36,18 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param context the application context * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If parameter is null, no audio source are set. It can be set later with [AudioOnlySingleStreamer.setAudioSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun AudioOnlySingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory = MicrophoneSourceFactory(), - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): AudioOnlySingleStreamer { val streamer = AudioOnlySingleStreamer( context = context, endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) return streamer @@ -54,16 +58,19 @@ suspend fun AudioOnlySingleStreamer( * * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class AudioOnlySingleStreamer( context: Context, - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IAudioSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = true, - withVideo = false + withVideo = false, + endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow @@ -74,8 +81,8 @@ class AudioOnlySingleStreamer( get() = streamer.info override val audioConfigFlow = streamer.audioConfigFlow - override val audioInput: IAudioInput = streamer.audioInput!! - + override val audioInput: IAudioInput = streamer.audioInput + override val audioEncoder: IEncoder? get() = streamer.audioEncoder @@ -97,12 +104,4 @@ class AudioOnlySingleStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() - - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - throw UnsupportedOperationException("Audio single streamer does not support bitrate regulator controller") - } - - override fun removeBitrateRegulatorController() { - // Do nothing - } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt index d19ffbfc3..f0d14fe8c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt @@ -55,22 +55,25 @@ interface ISingleStreamer : IOpenableStreamer { * Gets configuration information */ fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo +} +val ISingleStreamer.withAudio: Boolean /** - * Adds a bitrate regulator controller to the streamer. + * Whether the streamer has an audio source. */ - fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + get() = this is IAudioSingleStreamer +val ISingleStreamer.withVideo: Boolean /** - * Removes the bitrate regulator controller from the streamer. + * Whether the streamer has a video source. */ - fun removeBitrateRegulatorController() -} + get() = this is IVideoSingleStreamer + /** * An audio single Streamer */ -interface IAudioSingleStreamer : IAudioStreamer { +interface IAudioSingleStreamer : IAudioStreamer, ISingleStreamer { /** * The audio configuration flow. */ @@ -85,7 +88,7 @@ interface IAudioSingleStreamer : IAudioStreamer { /** * A video single streamer. */ -interface IVideoSingleStreamer : IVideoStreamer { +interface IVideoSingleStreamer : IVideoStreamer, ISingleStreamer { /** * The video configuration flow. */ @@ -95,5 +98,15 @@ interface IVideoSingleStreamer : IVideoStreamer { * Advanced settings for the video encoder. */ val videoEncoder: IEncoder? + + /** + * Adds a bitrate regulator controller to the streamer. + */ + fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + + /** + * Removes the bitrate regulator controller from the streamer. + */ + fun removeBitrateRegulatorController() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt index d7c1138da..6b3caa9eb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt @@ -25,7 +25,6 @@ import androidx.annotation.RequiresApi import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -35,7 +34,6 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MediaProjectionAudioSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue @@ -43,19 +41,10 @@ import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRo import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController -import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo -import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraSingleStreamer( @@ -73,10 +64,16 @@ suspend fun cameraSingleStreamer( cameraId: String = context.defaultCameraId, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( - context, withAudio = true, withVideo = true, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) if (audioSourceFactory != null) { @@ -92,20 +89,24 @@ suspend fun cameraSingleStreamer( * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -121,20 +122,24 @@ suspend fun audioVideoMediaProjectionSingleStreamer( * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -152,6 +157,8 @@ suspend fun videoMediaProjectionSingleStreamer( * @param videoSourceFactory the video source factory. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun SingleStreamer( context: Context, @@ -159,13 +166,15 @@ suspend fun SingleStreamer( videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, - withAudio = true, - withVideo = true, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) @@ -173,249 +182,102 @@ suspend fun SingleStreamer( } /** - * A [ISingleStreamer] implementation for audio and video. + * A [ISingleStreamer] implementation for both audio and video. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class SingleStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class SingleStreamer( + context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider(), ) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { - private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, - surfaceProcessorFactory, - dispatcherProvider + private val streamer = SingleStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - private val pipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, - withVideo, - endpointFactory, - defaultRotation - ) as IEncodingPipelineOutputInternal - } - - override val throwableFlow: StateFlow = - merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - override val isOpenFlow: StateFlow - get() = pipelineOutput.isOpenFlow - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // AUDIO - /** - * The audio input. - * It allows advanced audio source settings. - */ - override val audioInput = pipeline.audioInput + override val endpoint: IEndpoint + get() = streamer.endpoint + override val info: IConfigurationInfo + get() = streamer.info + override val audioConfigFlow = streamer.audioConfigFlow override val audioEncoder: IEncoder? - get() = pipelineOutput.audioEncoder - - override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = - pipeline.setAudioSource(audioSourceFactory) - - // VIDEO - /** - * The video input. - * It allows advanced video source settings. - */ - override val videoInput = pipeline.videoInput + get() = streamer.audioEncoder + override val audioInput: IAudioInput = streamer.audioInput!! + override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? - get() = pipelineOutput.videoEncoder - - // ENDPOINT - override val endpoint: IEndpoint - get() = pipelineOutput.endpoint + get() = streamer.videoEncoder + override val videoInput: IVideoInput = streamer.videoInput!! /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } - - /** - * Gets configuration information. - * - * Could throw an exception if the endpoint needs to infer the configuration from the - * [MediaDescriptor]. - * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. - */ - override val info: IConfigurationInfo - get() = if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpoint.info) - } else { - StreamerConfigurationInfo(endpoint.info) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Gets configuration information from [MediaDescriptor]. - * - * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is - * already known. - * - * @param descriptor the media descriptor - */ - override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { - val endpointInfo = try { - endpoint.info - } catch (_: Throwable) { - endpoint.getInfo(descriptor) - } - return if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpointInfo) - } else { - StreamerConfigurationInfo(endpointInfo) - } - } - - // CONFIGURATION - /** - * The audio configuration flow. - */ - override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow - - /** - * Configures audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * @param audioConfig Audio configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: AudioConfig) { - pipelineOutput.setAudioCodecConfig(audioConfig) - } + override suspend fun setAudioConfig(audioConfig: AudioConfig) = + streamer.setAudioConfig(audioConfig) - /** - * The video configuration flow. - */ - override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow - - /** - * Configures video settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ - override suspend fun setVideoConfig(videoConfig: VideoConfig) { - pipelineOutput.setVideoCodecConfig(videoConfig) - } + override suspend fun setVideoConfig(videoConfig: VideoConfig) = + streamer.setVideoConfig(videoConfig) - /** - * Configures both video and audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [IStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } + override fun getInfo(descriptor: MediaDescriptor) = streamer.getInfo(descriptor) - /** - * Opens the streamer endpoint. - * - * @param descriptor Media descriptor to open - */ - override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + override suspend fun open(descriptor: MediaDescriptor) = streamer.open(descriptor) - /** - * Closes the streamer endpoint. - */ - override suspend fun close() = pipelineOutput.close() + override suspend fun close() = streamer.close() - /** - * Starts audio/video stream. - * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote - * device. - * To avoid creating an unresponsive UI, do not call on main thread. - * - * @see [stopStream] - */ - override suspend fun startStream() = pipelineOutput.startStream() + override suspend fun startStream() = streamer.startStream() - /** - * Stops audio/video stream. - * - * Internally, it resets audio and video recorders and encoders to get them ready for another - * [startStream] session. It explains why preview could be restarted. - * - * @see [startStream] - */ - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun stopStream() = streamer.stopStream() - /** - * Releases the streamer. - */ - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } + override suspend fun release() = streamer.release() - /** - * Adds a bitrate regulator controller. - * - * Limitation: it is only available for SRT for now. - */ override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = - pipelineOutput.addBitrateRegulatorController(controllerFactory) + streamer.addBitrateRegulatorController(controllerFactory) - /** - * Removes the bitrate regulator controller. - */ - override fun removeBitrateRegulatorController() = - pipelineOutput.removeBitrateRegulatorController() + override fun removeBitrateRegulatorController() = streamer.removeBitrateRegulatorController() +} - companion object { - const val TAG = "SingleStreamer" - } + +/** + * Configures both video and audio settings. + * It is the first method to call after a [SingleStreamer] instantiation. + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [SingleStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun SingleStreamer.setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt new file mode 100644 index 000000000..e334ca746 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.single + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor +import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController +import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + +/** + * A [ISingleStreamer] implementation for audio and video. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + */ +internal class SingleStreamerImpl( + private val context: Context, + withAudio: Boolean, + withVideo: Boolean, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { + private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, + surfaceProcessorFactory, + dispatcherProvider + ) + private val pipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, + withVideo, + endpointFactory, + defaultRotation + ) as IEncodingPipelineOutputInternal + } + + override val throwableFlow: StateFlow = + merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + override val isOpenFlow: StateFlow + get() = pipelineOutput.isOpenFlow + + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + // AUDIO + /** + * The audio input. + * It allows advanced audio source settings. + */ + override val audioInput: IAudioInput + get() = pipeline.audioInput + + override val audioEncoder: IEncoder? + get() = pipelineOutput.audioEncoder + + override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = + pipeline.setAudioSource(audioSourceFactory) + + // VIDEO + /** + * The video input. + * It allows advanced video source settings. + */ + override val videoInput: IVideoInput + get() = pipeline.videoInput + + override val videoEncoder: IEncoder? + get() = pipelineOutput.videoEncoder + + // ENDPOINT + override val endpoint: IEndpoint + get() = pipelineOutput.endpoint + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Gets configuration information. + * + * Could throw an exception if the endpoint needs to infer the configuration from the + * [MediaDescriptor]. + * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. + */ + override val info: IConfigurationInfo + get() = if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpoint.info) + } else { + StreamerConfigurationInfo(endpoint.info) + } + + /** + * Gets configuration information from [MediaDescriptor]. + * + * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is + * already known. + * + * @param descriptor the media descriptor + */ + override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { + val endpointInfo = try { + endpoint.info + } catch (_: Throwable) { + endpoint.getInfo(descriptor) + } + return if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpointInfo) + } else { + StreamerConfigurationInfo(endpointInfo) + } + } + + // CONFIGURATION + /** + * The audio configuration flow. + */ + override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow + + /** + * Configures audio settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and audio capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * @param audioConfig Audio configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: AudioConfig) { + pipelineOutput.setAudioCodecConfig(audioConfig) + } + + /** + * The video configuration flow. + */ + override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow + + /** + * Configures video settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + override suspend fun setVideoConfig(videoConfig: VideoConfig) { + pipelineOutput.setVideoCodecConfig(videoConfig) + } + + /** + * Opens the streamer endpoint. + * + * @param descriptor Media descriptor to open + */ + override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + + /** + * Closes the streamer endpoint. + */ + override suspend fun close() = pipelineOutput.close() + + /** + * Starts audio/video stream. + * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote + * device. + * To avoid creating an unresponsive UI, do not call on main thread. + * + * @see [stopStream] + */ + override suspend fun startStream() = pipelineOutput.startStream() + + /** + * Stops audio/video stream. + * + * Internally, it resets audio and video recorders and encoders to get them ready for another + * [startStream] session. It explains why preview could be restarted. + * + * @see [startStream] + */ + override suspend fun stopStream() = pipeline.stopStream() + + /** + * Releases the streamer. + */ + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } + + /** + * Adds a bitrate regulator controller. + * + * Limitation: it is only available for SRT for now. + */ + override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = + pipelineOutput.addBitrateRegulatorController(controllerFactory) + + /** + * Removes the bitrate regulator controller. + */ + override fun removeBitrateRegulatorController() = + pipelineOutput.removeBitrateRegulatorController() + + companion object Companion { + const val TAG = "SingleStreamer" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt index fa8071750..c224ba146 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt @@ -26,12 +26,16 @@ import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -44,39 +48,53 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param cameraId the camera id to use. By default, it is the default camera. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlySingleStreamer( context: Context, cameraId: String = context.defaultCameraId, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( - context, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer } /** - * Creates a [SingleStreamer] with the screen as video source and no audio source. + * Creates a [VideoOnlySingleStreamer] with the screen as video source and no audio source. * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlySingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -89,17 +107,23 @@ suspend fun videoMediaProjectionVideoOnlySingleStreamer( * @param videoSourceFactory the video source factory. If parameter is null, no audio source are set. It can be set later with [VideoOnlySingleStreamer.setVideoSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlySingleStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -111,19 +135,27 @@ suspend fun VideoOnlySingleStreamer( * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlySingleStreamer( context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IVideoSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + endpointFactory = endpointFactory, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow @@ -136,7 +168,7 @@ class VideoOnlySingleStreamer( override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? get() = streamer.videoEncoder - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt index 2b8e3428b..3b96e15f9 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt @@ -6,7 +6,7 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.utils.DescriptorUtils -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.test.runTest import org.junit.Assert.assertFalse @@ -85,7 +85,7 @@ class DynamicEndpointTest { val dynamicEndpoint = DynamicEndpoint(context, Dispatchers.Default, Dispatchers.IO) try { dynamicEndpoint.write( - FakeFramesWithCloseable.create(MediaFormat.MIMETYPE_AUDIO_AAC), + FakeFrames.create(MediaFormat.MIMETYPE_AUDIO_AAC), 0 ) fail("Throwable expected") diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt index 1a5b3dc8d..b3fa4dd23 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.TSConst import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.Utils.createFakeServiceInfo -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -193,7 +193,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -206,7 +206,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -231,11 +231,11 @@ class TsMuxerTest { tsMux.addStreams(service, listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) } @@ -251,10 +251,10 @@ class TsMuxerTest { tsMux.addStreams(createFakeServiceInfo(), listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrames.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt index 2c579cdcd..393c6a473 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt @@ -18,11 +18,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.media.MediaRecorder import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSource import io.github.thibaultbee.streampack.core.elements.utils.StubLogger -import io.github.thibaultbee.streampack.core.elements.utils.StubRawFrameFactory import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.test.runTest import org.junit.Assert import org.junit.Test +import java.nio.ByteBuffer class MicrophoneSourceUnitTest { init { @@ -38,7 +38,7 @@ class MicrophoneSourceUnitTest { } catch (_: Throwable) { } try { - microphoneSource.getAudioFrame(StubRawFrameFactory()) + microphoneSource.fillAudioFrame(ByteBuffer.allocate(microphoneSource.minBufferSize)) Assert.fail() } catch (_: Throwable) { } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt index 83139d45c..542ef0a8e 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt @@ -16,8 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.MutableFrame import java.nio.ByteBuffer import kotlin.random.Random @@ -51,36 +52,19 @@ object FakeFrames { ) ) } - return Frame( + return MutableFrame( buffer, pts, dts, isKeyFrame, - listOf( - ByteBuffer.wrap( - Random.nextBytes(10) + Extra( + listOf( + ByteBuffer.wrap( + Random.nextBytes(10) + ) ) ), format = format ) } } - -object FakeFramesWithCloseable { - fun create( - mimeType: String, - buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), - pts: Long = Random.nextLong(), - dts: Long? = null, - isKeyFrame: Boolean = false - ) = FrameWithCloseable( - FakeFrames.create( - mimeType, - buffer, - pts, - dts, - isKeyFrame - ), - {/* Nothing to do */ } - ) -} \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt deleted file mode 100644 index a4d24568a..000000000 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IGetOnlyBufferPool -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory -import java.nio.ByteBuffer - -/** - * Stub buffer pool for testing. - * - * It always returns a new allocated buffer. - */ -class StubRawFrameFactory(private val bufferPool: IGetOnlyBufferPool = StubBufferPool()) : - IReadOnlyRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - // Do nothing - } - } -} \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 9825800e9..489364031 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -117,7 +117,7 @@ class ByteBufferExtensionsKtTest { ) testBuffer.position(2) - val clonedBuffer = testBuffer.clone() + val clonedBuffer = testBuffer.deepCopy() assertArrayEquals( testBuffer.toByteArray(), clonedBuffer.toByteArray() ) @@ -134,12 +134,16 @@ class ByteBufferExtensionsKtTest { val resultBuffers = testBuffer.slices( byteArrayOf(0, 0, 0, 1) ) + assertEquals( 3, resultBuffers.size ) - var resultArray = ByteArray(0) - resultBuffers.forEach { resultArray += it.array() } // concat all arrays - assertArrayEquals(testBuffer.array(), resultArray) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -172,7 +176,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals(testBuffer.array(), resultBuffers[0].array()) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -189,9 +198,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals( - testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffers[0].array() - ) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffer.array()) } @Test @@ -263,6 +275,10 @@ class ByteBufferExtensionsKtTest { ) ) val expectedArray = byteArrayOf( + 0, + 0, + 0, + 1, 66, 1, 1, @@ -336,4 +352,19 @@ class ByteBufferExtensionsKtTest { assertEquals(0, prefixBuffer.position()) assertEquals(0, testBuffer.position()) } + + @Test + fun `slice from to test`() { + val testBuffer = ByteBuffer.wrap("ABCDE".toByteArray()) + val slice = testBuffer.slice(1, 4) + + assertArrayEquals( + "BCDE".toByteArray(), slice.toByteArray() + ) + assertEquals(0, slice.position()) + assertEquals(4, slice.limit()) + + assertEquals(0, testBuffer.position()) + assertEquals(5, testBuffer.limit()) + } } \ No newline at end of file diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index acd8f2a14..a7d2d9aca 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -34,6 +34,7 @@ import io.github.thibaultbee.streampack.app.R import io.github.thibaultbee.streampack.app.databinding.MainFragmentBinding import io.github.thibaultbee.streampack.app.utils.DialogUtils import io.github.thibaultbee.streampack.app.utils.PermissionManager +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerViewModelLifeCycleObserver @@ -182,15 +183,17 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { private fun inflateStreamerPreview(streamer: IWithVideoSource) { val preview = binding.preview // Set camera settings button when camera is started - preview.listener = object : PreviewView.Listener { + preview.listener = object : PreviewView.PreviewListener { override fun onPreviewStarted() { Log.i(TAG, "Preview started") } + } - override fun onZoomRationOnPinchChanged(zoomRatio: Float) { - previewViewModel.onZoomRationOnPinchChanged() + preview.setZoomListener(object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + previewViewModel.onZoomChanged() } - } + }) // Wait till streamer exists to set it to the SurfaceView. lifecycleScope.launch { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index bf57eed0c..9dcb6c070 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -55,7 +55,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio +import io.github.thibaultbee.streampack.core.streamers.single.withVideo import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrtBitrateRegulatorController import kotlinx.coroutines.CancellationException @@ -79,16 +84,18 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod private val defaultDispatcher = Dispatchers.IO - private val buildStreamerUseCase = BuildStreamerUseCase(application, storageRepository) + private val buildStreamerUseCase = BuildStreamerUseCase(application) private val streamerFlow = - MutableStateFlow( - SingleStreamer( - application, - runBlocking { storageRepository.isAudioEnableFlow.first() }) // TODO avoid runBlocking + MutableStateFlow( + buildFirstStreamer(runBlocking { storageRepository.isAudioEnableFlow.first() }) ) - private val streamer: SingleStreamer + + private val streamer: IVideoSingleStreamer get() = streamerFlow.value + private val audioStreamer: IAudioSingleStreamer? + get() = streamer as? IAudioSingleStreamer? + val streamerLiveData = streamerFlow.asLiveData() /** @@ -111,10 +118,10 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val requiredPermissions: List get() { val permissions = mutableListOf() - if (streamer.videoInput?.sourceFlow is ICameraSource) { + if (streamer.videoInput.sourceFlow is ICameraSource) { permissions.add(Manifest.permission.CAMERA) } - if (streamer.audioInput?.sourceFlow?.value is IAudioRecordSource) { + if (audioStreamer?.audioInput?.sourceFlow?.value is IAudioRecordSource) { permissions.add(Manifest.permission.RECORD_AUDIO) } storageRepository.endpointDescriptorFlow.asLiveData().value?.let { @@ -149,7 +156,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // Set audio source and video source if (streamer.withAudio) { Log.i(TAG, "Audio source is enabled. Setting audio source") - streamer.setAudioSource(MicrophoneSourceFactory()) + audioStreamer!!.setAudioSource(MicrophoneSourceFactory()) } else { Log.i(TAG, "Audio source is disabled") } @@ -167,7 +174,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // TODO: cancel jobs linked to previous streamer viewModelScope.launch { - streamer.videoInput?.sourceFlow?.collect { + streamer.videoInput.sourceFlow.collect { notifySourceChanged() } } @@ -226,7 +233,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod ) == PackageManager.PERMISSION_GRANTED ) { try { - streamer.setAudioConfig(config) + audioStreamer!!.setAudioConfig(config) } catch (t: Throwable) { Log.e(TAG, "setAudioConfig failed", t) _streamerErrorLiveData.postValue("setAudioConfig: ${t.message ?: "Unknown error"}") @@ -247,7 +254,15 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - fun onZoomRationOnPinchChanged() { + private fun buildFirstStreamer(isAudioEnable: Boolean): IVideoSingleStreamer { + return if (isAudioEnable) { + SingleStreamer(application) + } else { + VideoOnlySingleStreamer(application) + } + } + + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } @@ -255,7 +270,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun configureAudio() { viewModelScope.launch { try { - storageRepository.audioConfigFlow.first()?.let { streamer.setAudioConfig(it) } + storageRepository.audioConfigFlow.first()?.let { audioStreamer?.setAudioConfig(it) } ?: Log.i( TAG, "Audio is disabled" @@ -271,7 +286,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun initializeVideoSource() { viewModelScope.launch { videoSourceMutex.withLock { - if (streamer.videoInput?.sourceFlow?.value == null) { + if (streamer.videoInput.sourceFlow.value == null) { streamer.setVideoSource(CameraSourceFactory(defaultCameraId)) } else { Log.i(TAG, "Camera source already set") @@ -327,7 +342,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } fun setMute(isMuted: Boolean) { - streamer.audioInput?.isMuted = isMuted + audioStreamer?.audioInput?.isMuted = isMuted } @RequiresPermission(Manifest.permission.CAMERA) @@ -337,7 +352,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod * exception instead of crashing. You can either catch the exception or check if the * configuration is valid for the new camera with [Context.isFpsSupported]. */ - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { @@ -357,7 +372,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod */ viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { streamer.setNextCameraId(application) } @@ -369,7 +384,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun toggleVideoSource() { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value val nextSource = when (videoSource) { is ICameraSource -> { BitmapSourceFactory(testBitmap) @@ -390,7 +405,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - val isCameraSource = streamer.videoInput?.sourceFlow?.map { it is ICameraSource }?.asLiveData() + val isCameraSource = streamer.videoInput.sourceFlow?.map { it is ICameraSource }?.asLiveData() val isFlashAvailable = MutableLiveData(false) fun toggleFlash() { @@ -514,7 +529,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } private fun notifySourceChanged() { - val videoSource = streamer.videoInput?.sourceFlow?.value ?: return + val videoSource = streamer.videoInput.sourceFlow.value ?: return if (videoSource is ICameraSource) { notifyCameraChanged(videoSource) } else { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt index 417f6614c..beef3a8f9 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt @@ -4,15 +4,17 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import androidx.core.app.ActivityCompat -import io.github.thibaultbee.streampack.app.data.storage.DataStoreRepository +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio class BuildStreamerUseCase( - private val context: Context, - private val dataStoreRepository: DataStoreRepository + private val context: Context ) { /** - * Build a new [SingleStreamer] based on audio and video preferences. + * Build a new [IVideoSingleStreamer] based on audio and video preferences. * * Only create a new streamer if the previous one is not the same type. * @@ -21,28 +23,37 @@ class BuildStreamerUseCase( * @param previousStreamer Previous streamer to check if we need to create a new one. */ suspend operator fun invoke( - previousStreamer: SingleStreamer, + previousStreamer: IVideoSingleStreamer, isAudioEnable: Boolean - ): SingleStreamer { + ): IVideoSingleStreamer { if (previousStreamer.withAudio != isAudioEnable) { - return SingleStreamer(context, isAudioEnable).apply { - // Get previous streamer config if any + previousStreamer.release() + + val streamer = if (isAudioEnable) { + SingleStreamer(context) + } else { + VideoOnlySingleStreamer(context) + } + + // Get previous streamer config if any + if ((previousStreamer is IAudioSingleStreamer) && (streamer is IAudioSingleStreamer)) { val audioConfig = previousStreamer.audioConfigFlow.value - val videoConfig = previousStreamer.videoConfigFlow.value - if ((audioConfig != null && isAudioEnable)) { + if (audioConfig != null) { if (ActivityCompat.checkSelfPermission( context, Manifest.permission.RECORD_AUDIO ) == PackageManager.PERMISSION_GRANTED ) { - setAudioConfig(audioConfig) + streamer.setAudioConfig(audioConfig) } } - if (videoConfig != null) { - setVideoConfig(videoConfig) - } - previousStreamer.release() } + + val videoConfig = previousStreamer.videoConfigFlow.value + if (videoConfig != null) { + streamer.setVideoConfig(videoConfig) + } + return streamer } return previousStreamer } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt index ec45c6943..bbe54ba7d 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt @@ -37,7 +37,7 @@ import io.github.thibaultbee.streampack.core.interfaces.setCameraId @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setNextCameraId(context: Context) { val cameras = context.cameraManager.cameras - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val newCameraId = if (videoSource is ICameraSource) { val currentCameraIndex = cameras.indexOf(videoSource.cameraId) @@ -52,7 +52,7 @@ suspend fun IWithVideoSource.setNextCameraId(context: Context) { @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.toggleBackToFront(context: Context) { val cameraManager = context.cameraManager - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val cameras = if (videoSource is ICameraSource) { if (cameraManager.isBackCamera(videoSource.cameraId)) { cameraManager.frontCameras diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt index 91bf48b07..661315379 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/FlvEndpoints.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.flv.encode import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.OnMetadata import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType @@ -126,12 +126,11 @@ sealed class FlvEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt index b0d1013b5..54df39c08 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt @@ -24,6 +24,7 @@ import io.github.komedia.komuxer.flv.tags.audio.ExtendedAudioDataFactory import io.github.komedia.komuxer.flv.tags.audio.codedFrame import io.github.komedia.komuxer.flv.tags.audio.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.AudioFlvMuxerInfo @@ -72,7 +73,7 @@ internal class FlvAudioDataFactory { val flvDatas = mutableListOf() if (withSequenceStart) { - val decoderConfigurationRecordBuffer = frame.extra!![0] + val decoderConfigurationRecordBuffer = frame.extra!!.get(0) flvDatas.add( aacAudioDataFactory.sequenceStart( decoderConfigurationRecordBuffer @@ -155,7 +156,7 @@ internal class FlvAudioDataFactory { return FlvExtendedAudioDataFactory( ExtendedAudioDataFactory(AudioFourCC.AAC), onSequenceStart = { frame -> - frame.extra!![0] + frame.extra!!.get(0) } ) } @@ -165,7 +166,7 @@ internal class FlvAudioDataFactory { ExtendedAudioDataFactory(AudioFourCC.OPUS), onSequenceStart = { frame -> frame.extra?.let { - OpusCsdParser.findIdentificationHeader(it[0]) + OpusCsdParser.findIdentificationHeader(it.get(0)) } } ) diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt index 14df85e5b..a450972d7 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.m import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.Metadata import io.github.komedia.komuxer.logger.KomuxerLogger -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -106,21 +106,19 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { } suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, ts: Int, streamPid: Int ) { if (ts < 0) { Logger.w( TAG, - "Negative timestamp $ts for frame ${closeableFrame.frame}. Frame will be dropped." + "Negative timestamp $ts for frame $frame. Frame will be dropped." ) - closeableFrame.close() + frame.close() return } - val frame = closeableFrame.frame - val flvDatas = when (streamPid) { AUDIO_STREAM_PID -> audioStream?.create(frame) ?: throw IllegalStateException("Audio stream not added") @@ -133,7 +131,7 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { flvDatas.forEachIndexed { index, flvData -> if (index == flvDatas.lastIndex) { // Pass the close callback on the last element - channel.send(FLVTag(ts, flvData), { closeableFrame.close() }) + channel.send(FLVTag(ts, flvData), { frame.close() }) } else { channel.send(FLVTag(ts, flvData)) } diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt index 6b541bc92..dccc05e0d 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt @@ -28,6 +28,8 @@ import io.github.komedia.komuxer.flv.tags.video.VideoFrameType import io.github.komedia.komuxer.flv.tags.video.codedFrame import io.github.komedia.komuxer.flv.tags.video.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord @@ -64,10 +66,11 @@ internal class FlvVideoDataFactory { VideoFrameType.INTER } if (frame.isKeyFrame && withSequenceStart) { + val extra = frame.extra!!.extra val decoderConfigurationRecordBuffer = AVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1] + extra[0], + extra[1] ).toByteBuffer() flvDatas.add( factory.sequenceStart( @@ -185,10 +188,11 @@ internal class FlvVideoDataFactory { private fun createHEVCFactory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(HEVCExtendedVideoDataFactory()) { frame -> // Extra is VPS, SPS, PPS + val extra = frame.extra!!.extra HEVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1], - frame.extra!![2] + extra[0], + extra[1], + extra[2] ).toByteBuffer() } } @@ -196,7 +200,7 @@ internal class FlvVideoDataFactory { private fun createAV1Factory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(ExtendedVideoDataFactory(VideoFourCC.AV1)) { frame -> // Extra is AV1CodecConfigurationRecord - frame.extra!![0] + frame.extra!!.get(0) } } diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt index b0d3d8eea..eab0e0eb9 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt @@ -22,7 +22,7 @@ import io.github.komedia.komuxer.rtmp.client.RtmpClient import io.github.komedia.komuxer.rtmp.connect import io.github.komedia.komuxer.rtmp.messages.command.StreamPublishType import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.ClosedException import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint @@ -164,12 +164,11 @@ class RtmpEndpoint internal constructor( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ba85fa48b..6ed29f24a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ videoApiClient = "1.6.7" androidxActivity = "1.10.1" androidxAppcompat = "1.7.1" androidxCamera = "1.4.0-alpha13" +androidxComposeBom = "2026.01.00" androidxConstraintlayout = "2.2.1" androidxCore = "1.17.0" androidxDatabinding = "8.13.0" @@ -29,6 +30,7 @@ robolectric = "4.16" komuxer = "0.3.3" srtdroid = "1.9.5" junitKtx = "1.3.0" +compose = "1.10.1" [libraries] android-documentation-plugin = { module = "org.jetbrains.dokka:android-documentation-plugin", version.ref = "dokka" } @@ -42,6 +44,7 @@ android-material = { module = "com.google.android.material:material", version.re androidx-activity = { module = "androidx.activity:activity", version.ref = "androidxActivity" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidxAppcompat" } androidx-camera-viewfinder-view = { module = "androidx.camera.viewfinder:viewfinder-view", version.ref = "androidxCamera" } +androidx-compose-bom = { module = "androidx.compose:compose-bom", version.ref = "androidxComposeBom" } androidx-concurrent-futures = { module = "androidx.concurrent:concurrent-futures", version.ref = "concurrentFutures" } androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "androidxConstraintlayout" } androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } @@ -70,10 +73,16 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re mockk = { module = "io.mockk:mockk", version.ref = "mockk" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } srtdroid-ktx = { module = "io.github.thibaultbee.srtdroid:srtdroid-ktx", version.ref = "srtdroid" } +androidx-compose-runtime = { group = "androidx.compose.runtime", name = "runtime", version.ref = "compose" } +androidx-compose-ui = { group = "androidx.compose.ui", name = "ui", version.ref = "compose" } +androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "compose" } +androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "compose" } +androidx-compose-foundation = { group = "androidx.compose.foundation", name = "foundation", version.ref = "compose" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } android-library = { id = "com.android.library", version.ref = "agp" } +compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt index 0b740e6b5..5f5888bfe 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt @@ -163,7 +163,7 @@ abstract class MediaProjectionService( } if (streamer is IWithVideoSource) { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is IMediaProjectionSource) { streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) } else if (videoSource == null) { diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 61f5bc703..d9866e2e4 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -16,13 +16,17 @@ package io.github.thibaultbee.streampack.services.utils import android.content.Context +import android.view.Surface import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.VideoOnlyDualStreamer +import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer import io.github.thibaultbee.streampack.services.StreamerService @@ -53,12 +57,19 @@ open class SingleStreamerFactory( ) : StreamerFactory { override fun create(context: Context): ISingleStreamer { - return SingleStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + SingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withAudio) { + AudioOnlySingleStreamer(context) + } else { + VideoOnlySingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } } } @@ -76,11 +87,18 @@ open class DualStreamerFactory( ) : StreamerFactory { override fun create(context: Context): IDualStreamer { - return DualStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + DualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withVideo) { + VideoOnlyDualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else { + throw IllegalArgumentException("DualStreamer audio only is not supported yet") + } } } \ No newline at end of file diff --git a/settings.libs.gradle.kts b/settings.libs.gradle.kts index c42addcde..7a8a4328c 100644 --- a/settings.libs.gradle.kts +++ b/settings.libs.gradle.kts @@ -1,11 +1,17 @@ // StreamPack libraries include(":core") project(":core").name = "streampack-core" -include(":ui") -project(":ui").name = "streampack-ui" include(":services") project(":services").name = "streampack-services" +// UI +include(":ui") +project(":ui").projectDir = File(rootDir, "ui/ui") +project(":ui").name = "streampack-ui" +include(":compose") +project(":compose").projectDir = File(rootDir, "ui/compose") +project(":compose").name = "streampack-compose" + // Extensions include(":extension-flv") project(":extension-flv").projectDir = File(rootDir, "extensions/flv") diff --git a/ui/.gitignore b/ui/compose/.gitignore similarity index 100% rename from ui/.gitignore rename to ui/compose/.gitignore diff --git a/ui/compose/build.gradle.kts b/ui/compose/build.gradle.kts new file mode 100644 index 000000000..de8c2b318 --- /dev/null +++ b/ui/compose/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + id(libs.plugins.android.library.get().pluginId) + id(libs.plugins.kotlin.android.get().pluginId) + id("android-library-convention") + alias(libs.plugins.compose.compiler) +} + +description = "Jetpack compose components for StreamPack." + +android { + namespace = "io.github.thibaultbee.streampack.compose" + + defaultConfig { + minSdk = 23 + } + buildFeatures { + compose = true + } +} + +dependencies { + implementation(project(":streampack-core")) + implementation(project(":streampack-ui")) + + val composeBom = platform(libs.androidx.compose.bom) + implementation(composeBom) + implementation(libs.androidx.compose.runtime) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.tooling.preview) + implementation(libs.androidx.compose.foundation) + + androidTestImplementation(composeBom) + + debugImplementation(libs.androidx.compose.ui.tooling) +} diff --git a/ui/proguard-rules.pro b/ui/compose/proguard-rules.pro similarity index 100% rename from ui/proguard-rules.pro rename to ui/compose/proguard-rules.pro diff --git a/ui/compose/src/main/AndroidManifest.xml b/ui/compose/src/main/AndroidManifest.xml new file mode 100644 index 000000000..cc947c567 --- /dev/null +++ b/ui/compose/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt new file mode 100644 index 000000000..379649bb7 --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/PreviewView.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose + +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.viewinterop.AndroidView +import io.github.thibaultbee.streampack.compose.utils.BitmapUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS +import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource +import io.github.thibaultbee.streampack.core.logger.Logger +import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.ui.views.PreviewView +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.withLock + +private const val TAG = "ComposePreviewView" + +/** + * Displays the preview of a [IWithVideoSource]. + * + * A [IWithVideoSource] must have a video sourc. + * + * @param videoSource the [IWithVideoSource] to preview + * @param modifier the [Modifier] to apply to the [PreviewView] + * @param enableZoomOnPinch enable zoom on pinch gesture + * @param enableTapToFocus enable tap to focus + * @param onTapToFocusTimeoutMs the duration in milliseconds after which the focus area set by tap-to-focus is cleared + */ +@Composable +fun PreviewScreen( + videoSource: IWithVideoSource, + modifier: Modifier = Modifier, + enableZoomOnPinch: Boolean = true, + enableTapToFocus: Boolean = true, + onTapToFocusTimeoutMs: Long = DEFAULT_AUTO_CANCEL_DURATION_MS +) { + val scope = rememberCoroutineScope() + + AndroidView( + factory = { context -> + PreviewView(context).apply { + this.enableZoomOnPinch = enableZoomOnPinch + this.enableTapToFocus = enableTapToFocus + this.onTapToFocusTimeoutMs = onTapToFocusTimeoutMs + + scope.launch { + try { + setVideoSourceProvider(videoSource) + } catch (e: Exception) { + Logger.e(TAG, "Failed to start preview", e) + } + } + } + }, + modifier = modifier, + onRelease = { + scope.launch { + val source = videoSource.videoInput?.sourceFlow?.value as? IPreviewableSource + source?.previewMutex?.withLock { + source.stopPreview() + source.resetPreview() + } + } + }) +} + +@Preview +@Composable +fun PreviewScreenPreview() { + val context = LocalContext.current + val streamer = SingleStreamer(context) + LaunchedEffect(Unit) { + streamer.setVideoSource( + BitmapSourceFactory( + BitmapUtils.createImage( + 1280, + 720 + ) + ) + ) + } + + PreviewScreen(streamer, modifier = Modifier.fillMaxSize()) +} \ No newline at end of file diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt new file mode 100644 index 000000000..c1cfec02b --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose.utils + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint + +object BitmapUtils { + /** + * Creates a bitmap with the given width, height and color. + * + * @param width The width of the bitmap. + * @param height The height of the bitmap. + * @param color The color of the bitmap. + * @return The created bitmap. + */ + fun createImage(width: Int, height: Int, color: Int = Color.RED): Bitmap { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val paint = Paint().apply { + this.color = color + } + canvas.drawRect(0f, 0f, width.toFloat(), height.toFloat(), paint) + return bitmap + } +} diff --git a/ui/ui/.gitignore b/ui/ui/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/ui/ui/.gitignore @@ -0,0 +1 @@ +/build diff --git a/ui/build.gradle.kts b/ui/ui/build.gradle.kts similarity index 100% rename from ui/build.gradle.kts rename to ui/ui/build.gradle.kts diff --git a/ui/ui/proguard-rules.pro b/ui/ui/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/ui/ui/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/ui/src/main/AndroidManifest.xml b/ui/ui/src/main/AndroidManifest.xml similarity index 100% rename from ui/src/main/AndroidManifest.xml rename to ui/ui/src/main/AndroidManifest.xml diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt similarity index 92% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 88dc5a1eb..8075db37f 100644 --- a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -35,6 +35,7 @@ import androidx.camera.viewfinder.core.ScaleType import androidx.camera.viewfinder.core.ViewfinderSurfaceRequest import androidx.camera.viewfinder.core.populateFromCharacteristics import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.getCameraCharacteristics @@ -108,9 +109,11 @@ class PreviewView @JvmOverloads constructor( } /** - * The [Listener] to listen to specific view events. + * The [PreviewListener] to listen to specific view events. */ - var listener: Listener? = null + var listener: PreviewListener? = null + + private var zoomListener: CameraSettings.Zoom.OnZoomChangedListener? = null private var touchUpEvent: MotionEvent? = null @@ -160,8 +163,8 @@ class PreviewView @JvmOverloads constructor( streamer: IWithVideoSource ) { sourceJob += defaultScope.launch { - streamer.videoInput?.sourceFlow?.runningHistoryNotNull() - ?.collect { (previousVideoSource, newVideoSource) -> + streamer.videoInput.sourceFlow.runningHistoryNotNull() + .collect { (previousVideoSource, newVideoSource) -> if (previousVideoSource == newVideoSource) { Logger.w(TAG, "No change in video source") } else { @@ -176,12 +179,46 @@ class PreviewView @JvmOverloads constructor( } if (newVideoSource is IPreviewableSource) { attachToStreamerIfReady(true) + zoomListener?.let { + registerZoomListener(it) + } } } } } } + /** + * Sets the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + * + * @param listener the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + */ + fun setZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener?) { + if (listener == null) { + unregisterZoomListener() + } else { + registerZoomListener(listener) + } + zoomListener = listener + } + + private fun registerZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener) { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.addListener(listener) + } + } + + private fun unregisterZoomListener() { + zoomListener?.let { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.removeListener(it) + } + } + } + + /** * Sets the [IWithVideoSource] to preview. * @@ -411,7 +448,7 @@ class PreviewView @JvmOverloads constructor( private suspend fun stopPreview() { streamer?.let { - val videoSource = it.videoInput?.sourceFlow?.value + val videoSource = it.videoInput.sourceFlow.value if (videoSource is IPreviewableSource) { Logger.d(TAG, "Stopping preview") videoSource.previewMutex.withLock { @@ -504,7 +541,6 @@ class PreviewView @JvmOverloads constructor( mutex.withLock { val zoom = source.settings.zoom zoom.onPinch(scaleFactor) - listener?.onZoomRationOnPinchChanged(zoom.getZoomRatio()) } } return true @@ -518,7 +554,7 @@ class PreviewView @JvmOverloads constructor( /** * A listener for the [PreviewView]. */ - interface Listener { + interface PreviewListener { /** * Called when the preview is started. */ @@ -528,12 +564,6 @@ class PreviewView @JvmOverloads constructor( * Called when the preview failed to start. */ fun onPreviewFailed(t: Throwable) {} - - /** - * Called when the zoom ratio is changed. - * @param zoomRatio the new zoom ratio - */ - fun onZoomRationOnPinchChanged(zoomRatio: Float) {} } /** diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt similarity index 91% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt index 7134c77cd..68a4efd41 100644 --- a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt @@ -18,7 +18,7 @@ suspend fun IWithVideoSource.setPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.setPreview(viewfinder, previewSize) } @@ -35,7 +35,7 @@ suspend fun IWithVideoSource.startPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.startPreview(viewfinder, previewSize) } diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt diff --git a/ui/src/main/res/values/attrs.xml b/ui/ui/src/main/res/values/attrs.xml similarity index 100% rename from ui/src/main/res/values/attrs.xml rename to ui/ui/src/main/res/values/attrs.xml