To use the model for transcription, use the system prompt Perform ASR in <language>., where <language> is the English name of the language.
To let the model infer the language, use Perform ASR..

Below, we transcribe a .wav file containing English speech.

conversation = [
    {"role": "system", "content": "Perform ASR in English."},
    {"role": "user", "content": [{"type": "audio", "audio": "audio.wav"}]},
]

inputs = processor.apply_chat_template(
    conversation,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(device)
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=1024,
        suppress_mixed=True,
        suppress_audio_out=True,
        return_dict_in_generate=True,
    )
text = processor.decode(
    outputs.sequences[0, inputs["input_ids"].shape[1]:],
    skip_special_tokens=True,
)
print(text)

LiveASR transcribes speech while it is still being spoken, instead of waiting for the full recording. Use the system prompt Perform live ASR.. LiveASR currently supports English only. The model processes audio in chunks of about 0.96 s and writes the text for each chunk as soon as the chunk arrives. model.transcribe_live() runs one chunk at a time and returns a state that you pass to the next call.

Below, we simulate a microphone by feeding a .wav file in 100 ms pieces.

import librosa

tokenizer = processor.tokenizer
chunk_frames = 8 * model.config.interleaved_n_audio
prompt = tokenizer(
    "<|startoftext|><|im_start|>system\nPerform live ASR.<|im_end|>\n<|im_start|>assistant\n",
    add_special_tokens=False,
    return_tensors="pt",
).input_ids.to(device)
suppress = tokenizer.convert_tokens_to_ids(["<|mixed_end|>", "<|endoftext|>"])
stop = tokenizer.convert_tokens_to_ids(["<|endoftext|>", "<|im_start|>"])
state, feature_cache = None, None
mel = torch.empty((1, processor.feature_extractor.feature_size, 0), device=device)


def transcribe(frames, is_final=False):
    global state
    output = model.transcribe_live(
        frames,
        state,
        input_ids=prompt if state is None else None,
        is_final=is_final,
        max_new_tokens=96 if is_final else 48,
        suppress_token_ids=suppress,
        stop_token_ids=stop,
    )
    state = output.state
    print(tokenizer.decode(output.sequences[0], skip_special_tokens=True), end="", flush=True)


waveform, _ = librosa.load("audio.wav", sr=16_000)
for start in range(0, len(waveform), 1_600):  # 100 ms microphone chunks
    features, feature_cache = processor.feature_extractor.stream(
        waveform[start:start + 1_600], feature_cache
    )
    mel = torch.cat((mel, features.input_features.to(device)), dim=-1)
    while mel.shape[-1] >= chunk_frames:
        transcribe(mel[..., :chunk_frames])
        mel = mel[..., chunk_frames:]

# flush the remaining audio and end the transcript
features, feature_cache = processor.feature_extractor.stream([], feature_cache, is_final=True)
mel = torch.cat((mel, features.input_features.to(device)), dim=-1)
while mel.shape[-1] > chunk_frames:
    transcribe(mel[..., :chunk_frames])
    mel = mel[..., chunk_frames:]
transcribe(mel, is_final=True)
print()

For streams longer than 30 seconds, pass input_ids=prompt again to start a new text context. The encoder keeps following the audio stream.

For speech synthesis, use the system prompt Perform TTS in <language> using a <gender> voice..
Use a language or locale from the model details above and a female or male voice. Prefill the assistant turn with <|audio_start|>.

conversation = [
    {"role": "system", "content": "Perform TTS in American English using a female voice."},
    {"role": "user", "content": "The quick brown fox jumps over the lazy dog"},
    {"role": "assistant", "content": "<|audio_start|>"},
]
inputs = processor.apply_chat_template(
    conversation,
    continue_final_message=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(device)
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=256,
        audio_temperature=1.0,
        audio_top_k=128,
        audio_repetition_penalty=1.1,
        audio_repetition_window=4096,
        return_dict_in_generate=True,
    )
    waveform = codec.decode(
        outputs.audio_codes[:1, :, :outputs.audio_lengths[0]]
    ).audio_values[0]  # 24 kHz

sf.write("tts.wav", waveform.float().cpu().numpy(), 24_000)

In this mode, the model will respond to the user in both speech and text.
The recommended system prompt is Respond with interleaved text and audio..
User input can be either text or audio, and we prefill the assistant turn with a special token <|mixed_start|>.

The assistant responds in a conversational style with interleaved text and speech. This example saves the complete response.

conversation = [
    {"role": "system", "content": "Respond with interleaved text and audio."},
    {"role": "user", "content": [{"type": "audio", "audio": "question.wav"}]},
    {"role": "assistant", "content": "<|mixed_start|>"},
]
inputs = processor.apply_chat_template(
    conversation,
    continue_final_message=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(device)
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=1024,
        audio_temperature=1.0,
        audio_top_k=128,
        audio_repetition_penalty=1.1,
        audio_repetition_window=4096,
        return_dict_in_generate=True,
    )

    text = processor.decode(
        outputs.sequences[0, inputs["input_ids"].shape[1]:],
        skip_special_tokens=True,
    )
    waveform = codec.decode(
        outputs.audio_codes[:1, :, :outputs.audio_lengths[0]]
    ).audio_values[0]  # 24 kHz
print(text)
sf.write("reply.wav", waveform.float().cpu().numpy(), 24_000)

For streaming applications, model.generate() accepts streamer for text tokens and audio_streamer for speech frames. LiquidCodec supports incremental decoding of those frames.

For a text-only answer, no specific system prompt is needed. Prefill <think> to let the model reason before answering, or <think></think> for a direct response.

use_thinking = True
conversation = [
    {"role": "user", "content": [{"type": "audio", "audio": "question.wav"}]},
    {"role": "assistant", "content": "<think>" if use_thinking else "<think></think>"},
]
inputs = processor.apply_chat_template(
    conversation,
    continue_final_message=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(device)
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=1024,
        suppress_mixed=True,
        suppress_audio_out=True,
        return_dict_in_generate=True,
    )
    text = processor.decode(
        outputs.sequences[0, inputs["input_ids"].shape[1]:],
        skip_special_tokens=True,
    )
if use_thinking:
    _, separator, text = text.partition("</think>")
    if not separator:
        raise RuntimeError("Reasoning has not finished; increase max_new_tokens.")
print(text.strip())

Include the recording and a question in the user turn. Questions can address speech, music, background sounds, or events. No specific system prompt is required; to ground answers in the recording, use Answer questions about the provided audio clip. Base every answer only on what can be heard..

conversation = [
    {
        "role": "system",
        "content": "Answer questions about the provided audio clip. Base every answer only on what can be heard.",
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "recording.wav"},
            {"type": "text", "text": "What is happening in this recording?"},
        ],
    },
]
inputs = processor.apply_chat_template(
    conversation,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(device)

with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=1024,
        suppress_mixed=True,
        suppress_audio_out=True,
        return_dict_in_generate=True,
    )

print(processor.decode(
    outputs.sequences[0, inputs["input_ids"].shape[1]:],
    skip_special_tokens=True,
))

For multiple-choice questions, include the choices in the text part and ask the model to select an answer.

Use a system prompt that specifies the source language, target language, and output format:

• Speech to text: Translate the following speech from {source} to {target} in text form.
• Text to text: Translate the following {source} text into {target}.

Replace {source} and {target} with the English names of the source and target languages. Follow the ASR example with the translation prompt; for text input, set the user message's content to the text to translate. Translation supports the speech languages listed above; translation to or from English generally works best.

Pass function schemas through tools=... to apply_chat_template(). The chat template appends them to the system prompt, one JSON definition per line inside <tools>…</tools>. Requests may be spoken or written.

With the system prompt Respond with function calls, and interleaved text and audio.\n\nList of tools:, the model either calls tools or answers with interleaved text and speech. Do not prefill the assistant turn: the model chooses whether to start with <tool_call> or <|mixed_start|>. A tool call looks like this, with multiple calls separated by a blank line:

<tool_call>
<function=set_volume>
<parameter=volume>
30
</parameter>
</function>
</tool_call>

Values other than strings are written as JSON. Your application parses and runs the calls, then appends the assistant's tool_calls (with arguments as a dict) and a tool message with the results to the conversation. The template wraps the results in <tool_response>…</tool_response>. Generate again with the same tools: the model either makes more calls or answers.

import json
import re

tools = [{
    "type": "function",
    "function": {
        "name": "set_volume",
        "description": "Set the device speaker volume.",
        "parameters": {
            "type": "object",
            "properties": {
                "volume": {"type": "integer", "description": "Volume from 0 to 100."}
            },
            "required": ["volume"],
        },
    },
}]
functions = {"set_volume": lambda volume: {"volume": volume}}  # your implementations

CALL_RE = re.compile(r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.DOTALL)
PARAM_RE = re.compile(r"<parameter=([^>\n]+)>\n?(.*?)\n?</parameter>", re.DOTALL)


def parse_tool_calls(text):
    calls = []
    for name, body in CALL_RE.findall(text):
        arguments = {}
        for key, value in PARAM_RE.findall(body):
            try:
                arguments[key] = json.loads(value)
            except json.JSONDecodeError:
                arguments[key] = value  # plain string
        calls.append({"type": "function", "function": {"name": name, "arguments": arguments}})
    return calls


def respond(conversation):
    inputs = processor.apply_chat_template(
        conversation,
        tools=tools,
        add_generation_prompt=True,
        tokenize=True,
        return_dict=True,
        return_tensors="pt",
    ).to(device)
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=1024,
            audio_temperature=1.0,
            audio_top_k=128,
            audio_repetition_penalty=1.1,
            audio_repetition_window=4096,
            return_dict_in_generate=True,
        )
    text = processor.decode(
        outputs.sequences[0, inputs["input_ids"].shape[1]:],
        skip_special_tokens=True,
    )
    return text, outputs


conversation = [
    {
        "role": "system",
        "content": "Respond with function calls, and interleaved text and audio.\n\nList of tools:",
    },
    {"role": "user", "content": [{"type": "audio", "audio": "question.wav"}]},
]
text, outputs = respond(conversation)
while calls := parse_tool_calls(text):
    results = [functions[c["function"]["name"]](**c["function"]["arguments"]) for c in calls]
    conversation += [
        {"role": "assistant", "tool_calls": calls},
        {"role": "tool", "content": json.dumps(results)},
    ]
    text, outputs = respond(conversation)

print(text)
with torch.no_grad():
    waveform = codec.decode(
        outputs.audio_codes[:1, :, :outputs.audio_lengths[0]]
    ).audio_values[0]  # 24 kHz
sf.write("reply.wav", waveform.float().cpu().numpy(), 24_000)

Text-only answers: use the system prompt List of tools: (keep tools=) and pass suppress_mixed=True, suppress_audio_out=True to generate(). Calls and results are handled the same way.

English ASR: average word error rate (WER) in percent on the ESB / Open ASR benchmark suite, lower is better.
Multilingual ASR: mean error rate in percent on FLEURS across English, Spanish, Portuguese, German, French, Italian, Chinese, Japanese, and Korean, lower is better. WER is used for all languages except Chinese, Japanese, and Korean, which use character error rate (CER).
English TTS: UTMOSv2 speech quality score on a 1–5 scale, higher is better.
VoiceBench: audio input, text output, percentage score, higher is better.
Tool calling: internal evaluation with audio input, percentage score, higher is better.
Audio understanding: MMAR, percentage score, higher is better.

Note: Evaluations used audio_temperature=0.4, audio_top_k=0, audio_repetition_penalty=1.2 and audio_repetition_window=64, with greedy text decoding except VoiceBench (do_sample=True, temperature=0.1, top_k=0, repetition_penalty=1.1, text_repetition_window=2048).