speech continuation examples

[adibrastegarniagithub]

Is there any examples to show how we can reproduce speech continuation examples from demo page?

[SumirSeth]

Hey @adibrastegarniagithub ! did you find anything useful?

[adibrastegarniagithub]

I have not tested yet completely but at very high level I guess it should be something like this

#!/usr/bin/env python3
"""MiMo-Audio Speech Continuation"""

from src.mimo_audio.process_speechdata import InputSegment
from src.mimo_audio.modeling_mimo_audio import MiMoStopper, MiMoSampler


def continue_speech(model_base, prompt_audio_path: str, output_path: str, max_new_tokens: int = 2048):
    """
    Continue speech from audio prompt.

    Args:
        model_base: MiMoAudio.from_pretrained("BrainPort/MiMo-Audio-base")
        prompt_audio_path: Audio to continue
        output_path: Output file
        max_new_tokens: 2048≈5-10s, 4096≈10-20s, 8192≈20-40s
    """
    audio_tokenized = model_base.preprocess_input(prompt_audio_path)

    prompt = [
        InputSegment(
            audio=audio_tokenized,
            speech_zeroemb_idx=model_base.speech_zeroemb_idx,
            text_zeroemb_idx=model_base.empty_token,
            add_sosp_eosp=True,
        )
    ]

    input_ids = model_base.get_input_ids(prompt)

    stopping_criteria = [
        MiMoStopper(
            stop_tokens=[model_base.tokenizer.eos_token_id, model_base.eostm_idx],
            group_size=model_base.group_size,
            audio_channels=model_base.audio_channels,
        )
    ]

    model_base.task_sampler_configs["continuation"] = {
        "global": MiMoSampler(do_sample=True, temperature=0.5, top_k=40, top_p=0.92),
        "local": MiMoSampler(do_sample=True, temperature=0.7, top_k=40, top_p=0.92)
    }

    model_base.forward(
        input_ids,
        output_audio_path=output_path,
        stopping_criteria=stopping_criteria,
        max_new_tokens=max_new_tokens,
        min_new_tokens=0,
        task_name="continuation"
    )

    return output_path


# Example:
"""
from src.mimo_audio.modeling_mimo_audio import MiMoAudio

model = MiMoAudio.from_pretrained("BrainPort/MiMo-Audio-base")
continue_speech(model, "prompt.wav", "output.wav", max_new_tokens=4096)
"""

[SumirSeth]

hm, i'm not sure actually. i was inclining towards something like few-shot examples during in-context learning pathway.

Something like this:

from src.mimo_audio.mimo_audio import MimoAudio

model_path = "models/MiMo-Audio-7B-Base"
tokenizer_path = "models/MiMo-Audio-Tokenizer"


model = MimoAudio(model_path, tokenizer_path)


# in context learning: speech-to-speech generation
instruction = "Continue the speech preserving timbre, prosody and speaker characterisitics."

input_audio = "examples/ESD/0013_000200.wav"
prompt_examples = [
    {
        "input_audio": "examples/ESD/0013_000139.wav",
        "output_audio": "examples/ESD/0019_000139.wav",
        "output_transcription": "Cuckoos is downheaded and crying.",
    },
    {
        "input_audio": "examples/ESD/0013_000963.wav",
        "output_audio": "examples/ESD/0019_000963.wav",
        "output_transcription": "She said in subdued voice.",
    },
    {
        "input_audio": "examples/ESD/0013_000559.wav",
        "output_audio": "examples/ESD/0019_000559.wav",
        "output_transcription": "A raging fire was-in his eyes.",
    },
    {
        "input_audio": "examples/ESD/0013_001142.wav",
        "output_audio": "examples/ESD/0019_001142.wav",
        "output_transcription": "Does the one that wins get the crowned?",
    },
    {
        "input_audio": "examples/ESD/0013_000769.wav",
        "output_audio": "examples/ESD/0019_000769.wav",
        "output_transcription": "Not much use is it, sam?",
    },
]

output_audio_path = "examples/in_context_learning_s2s.wav"
text_channel_output = model.in_context_learning_s2s(instruction, prompt_examples, input_audio, max_new_tokens=8192, output_audio_path=output_audio_path)

And in the prompt examples, we give it examples of how to continue the speech.

---

What do you think?