Files
Qwen3-TTS-Rust-Fixed/src/tts/prompt.rs
T

395 lines
12 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use crate::assets_manager::Assets;
use crate::utils::tokenizer::Tokenizer;
// Constants from PROTOCOL
pub const PAD: usize = 2148;
pub const BOS: usize = 2149;
pub const EOS: usize = 2150;
pub const BOS_TOKEN: usize = 151672;
pub const EOS_TOKEN: usize = 151673;
pub const THINK: usize = 2154;
pub const NOTHINK: usize = 2155;
pub const THINK_BOS: usize = 2156;
pub const THINK_EOS: usize = 2157;
// Magic token derived from Python implementation (assets.text_table[151671])
pub const TEXT_AUDIO_MARKER: usize = 151671;
pub struct PromptData {
pub embd: Vec<Vec<f32>>,
pub text_ids: Vec<u32>,
pub spk_emb: Vec<f32>,
}
pub struct PromptBuilder;
impl PromptBuilder {
#[allow(clippy::too_many_arguments)]
pub fn build_clone_prompt(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
ref_codes: &[i32],
ref_text_ids: &[u32],
spk_emb: &[f32],
lang_id: usize,
instruct: Option<&str>,
) -> PromptData {
let tts_pad = &assets.tts_pad;
let text_ids = tokenizer.encode(text);
let mut text_pool_ids: Vec<u32> = ref_text_ids.to_vec();
text_pool_ids.extend(text_ids.iter().copied());
text_pool_ids.push(EOS_TOKEN as u32);
let text_pool: Vec<Vec<f32>> = text_pool_ids
.iter()
.map(|&id| assets.get_text_embedding(id as usize))
.collect();
let mut audio_pool: Vec<Vec<f32>> = Vec::new();
audio_pool.push(assets.get_codec_embedding(0, BOS as i32));
let n_steps = ref_codes.len() / 16;
for step in 0..n_steps {
let mut step_sum = vec![0.0f32; 2048];
for q in 0..16 {
let c = ref_codes[step * 16 + q];
let emb = assets.get_codec_embedding(q, c);
for i in 0..2048 {
step_sum[i] += emb[i];
}
}
audio_pool.push(step_sum);
}
let t_len = text_pool.len();
let a_len = audio_pool.len();
let max_len = t_len.max(a_len);
let mut body = Vec::with_capacity(max_len);
for i in 0..max_len {
let text_vec = if i < t_len { &text_pool[i] } else { tts_pad };
let audio_vec = if i < a_len { &audio_pool[i] } else { tts_pad };
let fused: Vec<f32> = text_vec
.iter()
.zip(audio_vec.iter())
.map(|(t, a)| t + a)
.collect();
body.push(fused);
}
Self::build_core_with_clone_body(
text,
tokenizer,
assets,
Some(lang_id),
Some(spk_emb),
instruct,
body,
)
}
pub fn build_custom_prompt(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
spk_id: usize,
lang_id: usize,
instruct: Option<&str>,
) -> Result<PromptData, String> {
Self::build_core(
text,
tokenizer,
assets,
Some(lang_id),
Some(spk_id),
None,
instruct,
None,
)
}
fn build_core_with_clone_body(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
lang_id: Option<usize>,
spk_emb: Option<&[f32]>,
instruct: Option<&str>,
body: Vec<Vec<f32>>,
) -> PromptData {
let mut embeds = Vec::new();
if let Some(ins) = instruct {
let prefix = vec![151644, 872, 198];
for id in prefix {
embeds.push(assets.get_text_embedding(id));
}
let content_ids = tokenizer.encode(ins);
for id in content_ids {
embeds.push(assets.get_text_embedding(id as usize));
}
let suffix = vec![151645, 198];
for id in suffix {
embeds.push(assets.get_text_embedding(id));
}
}
for id in [151644, 77091, 198] {
embeds.push(assets.get_text_embedding(id));
}
let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER);
if let Some(lid) = lang_id {
let ids = [THINK, THINK_BOS, lid, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
} else {
let ids = [NOTHINK, THINK_BOS, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
}
if let Some(se) = spk_emb {
let sum: Vec<f32> = marker_emb
.iter()
.zip(se.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
embeds.extend(body);
let text_ids = tokenizer.encode(text);
let result_spk_emb = spk_emb
.map(|s| s.to_vec())
.unwrap_or_else(|| vec![0.0; 2048]);
PromptData {
embd: embeds,
text_ids: text_ids.into_iter().collect(),
spk_emb: result_spk_emb,
}
}
#[allow(clippy::too_many_arguments)]
pub fn build_core(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
lang_id: Option<usize>,
spk_id: Option<usize>,
spk_emb: Option<&[f32]>,
instruct: Option<&str>,
mid_embeds: Option<Vec<Vec<f32>>>,
) -> Result<PromptData, String> {
let mut embeds = Vec::new();
let text_ids = tokenizer.encode(text);
// Check if text_ids is empty (e.g., only punctuation)
if text_ids.is_empty() {
return Err("Text is empty or contains only unsupported characters".to_string());
}
// Check if text contains only punctuation and whitespace
let text_trimmed: String = text.chars().filter(|c| !c.is_whitespace()).collect();
let only_punctuation = text_trimmed.chars().all(|c| {
matches!(
c,
',' | '。'
| '、'
| ';'
| ':'
| '?'
| '!'
| '"'
| '\''
| '('
| ')'
| '【'
| '】'
| '《'
| '》'
| '…'
| '—'
| '~'
| ','
| '.'
| '!'
| '?'
| ';'
| ':'
| '('
| ')'
| '['
| ']'
| '<'
| '>'
| '-'
| '~'
| '`'
| '@'
| '#'
| '$'
| '%'
| '^'
| '&'
| '*'
| '+'
| '='
| '|'
| '\\'
| '/'
| '{'
| '}'
| '_'
)
});
if only_punctuation && !text_trimmed.is_empty() {
return Err(
"Text contains only punctuation marks, which may cause generation issues"
.to_string(),
);
}
let result_spk_emb = spk_emb
.map(|s| s.to_vec())
.unwrap_or_else(|| vec![0.0; 2048]);
let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER);
// 1. Instruct Block (User)
if let Some(ins) = instruct {
// <|im_start|>user\n
let prefix = vec![151644, 872, 198];
for id in prefix {
embeds.push(assets.get_text_embedding(id));
}
let content_ids = tokenizer.encode(ins);
for id in content_ids {
embeds.push(assets.get_text_embedding(id as usize));
}
// <|im_end|>\n
let suffix = vec![151645, 198];
for id in suffix {
embeds.push(assets.get_text_embedding(id));
}
}
// 2. Role Block (Assistant)
// <|im_start|>assistant\n -> [151644, 77091, 198]
for id in [151644, 77091, 198] {
embeds.push(assets.get_text_embedding(id));
}
// 3. Control Block
if let Some(lid) = lang_id {
// THINK, THINK_BOS, lang_id, THINK_EOS
let ids = [THINK, THINK_BOS, lid, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
} else {
// NOTHINK, THINK_BOS, THINK_EOS
let ids = [NOTHINK, THINK_BOS, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
}
// Speaker ID/Emb
if let Some(sid) = spk_id {
let e = assets.get_codec_embedding(0, sid as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
} else if let Some(se) = spk_emb {
let sum: Vec<f32> = marker_emb
.iter()
.zip(se.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
// 4. Mid Embeds
if let Some(mids) = mid_embeds {
embeds.extend(mids);
}
// 5. Task Text Block
// BOS_TOKEN + PAD
let pad_0 = assets.get_codec_embedding(0, PAD as i32);
let bos_token_emb = assets.get_text_embedding(BOS_TOKEN);
let bos_sum: Vec<f32> = bos_token_emb
.iter()
.zip(pad_0.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(bos_sum);
for &id in &text_ids {
let t_emb = assets.get_text_embedding(id as usize);
let sum: Vec<f32> = t_emb.iter().zip(pad_0.iter()).map(|(a, b)| a + b).collect();
embeds.push(sum);
}
// EOS_TOKEN + PAD
let eos_token_emb = assets.get_text_embedding(EOS_TOKEN);
let eos_sum: Vec<f32> = eos_token_emb
.iter()
.zip(pad_0.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(eos_sum);
// 6. Activation (BOS)
// Python: assets.text_table[151671] + assets.emb_tables[0][p["BOS"]]
let bos_emb = assets.get_codec_embedding(0, BOS as i32);
let act_sum: Vec<f32> = marker_emb
.iter()
.zip(bos_emb.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(act_sum);
Ok(PromptData {
embd: embeds,
text_ids: text_ids.into_iter().collect(),
spk_emb: result_spk_emb,
})
}
}