use crate::assets_manager::Assets; use crate::utils::tokenizer::Tokenizer; // Constants from PROTOCOL pub const PAD: usize = 2148; pub const BOS: usize = 2149; pub const EOS: usize = 2150; pub const BOS_TOKEN: usize = 151672; pub const EOS_TOKEN: usize = 151673; pub const THINK: usize = 2154; pub const NOTHINK: usize = 2155; pub const THINK_BOS: usize = 2156; pub const THINK_EOS: usize = 2157; // Magic token derived from Python implementation (assets.text_table[151671]) pub const TEXT_AUDIO_MARKER: usize = 151671; pub struct PromptData { pub embd: Vec>, pub text_ids: Vec, pub spk_emb: Vec, } pub struct PromptBuilder; impl PromptBuilder { #[allow(clippy::too_many_arguments)] pub fn build_clone_prompt( text: &str, tokenizer: &Tokenizer, assets: &Assets, ref_codes: &[i32], ref_text_ids: &[u32], spk_emb: &[f32], lang_id: usize, instruct: Option<&str>, ) -> PromptData { let tts_pad = &assets.tts_pad; let text_ids = tokenizer.encode(text); let mut text_pool_ids: Vec = ref_text_ids.to_vec(); text_pool_ids.extend(text_ids.iter().copied()); text_pool_ids.push(EOS_TOKEN as u32); let text_pool: Vec> = text_pool_ids .iter() .map(|&id| assets.get_text_embedding(id as usize)) .collect(); let mut audio_pool: Vec> = Vec::new(); audio_pool.push(assets.get_codec_embedding(0, BOS as i32)); let n_steps = ref_codes.len() / 16; for step in 0..n_steps { let mut step_sum = vec![0.0f32; 2048]; for q in 0..16 { let c = ref_codes[step * 16 + q]; let emb = assets.get_codec_embedding(q, c); for i in 0..2048 { step_sum[i] += emb[i]; } } audio_pool.push(step_sum); } let t_len = text_pool.len(); let a_len = audio_pool.len(); let max_len = t_len.max(a_len); let mut body = Vec::with_capacity(max_len); for i in 0..max_len { let text_vec = if i < t_len { &text_pool[i] } else { tts_pad }; let audio_vec = if i < a_len { &audio_pool[i] } else { tts_pad }; let fused: Vec = text_vec .iter() .zip(audio_vec.iter()) .map(|(t, a)| t + a) .collect(); body.push(fused); } Self::build_core_with_clone_body( text, tokenizer, assets, Some(lang_id), Some(spk_emb), instruct, body, ) } pub fn build_custom_prompt( text: &str, tokenizer: &Tokenizer, assets: &Assets, spk_id: usize, lang_id: usize, instruct: Option<&str>, ) -> Result { Self::build_core( text, tokenizer, assets, Some(lang_id), Some(spk_id), None, instruct, None, ) } fn build_core_with_clone_body( text: &str, tokenizer: &Tokenizer, assets: &Assets, lang_id: Option, spk_emb: Option<&[f32]>, instruct: Option<&str>, body: Vec>, ) -> PromptData { let mut embeds = Vec::new(); if let Some(ins) = instruct { let prefix = vec![151644, 872, 198]; for id in prefix { embeds.push(assets.get_text_embedding(id)); } let content_ids = tokenizer.encode(ins); for id in content_ids { embeds.push(assets.get_text_embedding(id as usize)); } let suffix = vec![151645, 198]; for id in suffix { embeds.push(assets.get_text_embedding(id)); } } for id in [151644, 77091, 198] { embeds.push(assets.get_text_embedding(id)); } let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER); if let Some(lid) = lang_id { let ids = [THINK, THINK_BOS, lid, THINK_EOS]; for &id in &ids { let e = assets.get_codec_embedding(0, id as i32); let sum: Vec = marker_emb .iter() .zip(e.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } } else { let ids = [NOTHINK, THINK_BOS, THINK_EOS]; for &id in &ids { let e = assets.get_codec_embedding(0, id as i32); let sum: Vec = marker_emb .iter() .zip(e.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } } if let Some(se) = spk_emb { let sum: Vec = marker_emb .iter() .zip(se.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } embeds.extend(body); let text_ids = tokenizer.encode(text); let result_spk_emb = spk_emb .map(|s| s.to_vec()) .unwrap_or_else(|| vec![0.0; 2048]); PromptData { embd: embeds, text_ids: text_ids.into_iter().collect(), spk_emb: result_spk_emb, } } #[allow(clippy::too_many_arguments)] pub fn build_core( text: &str, tokenizer: &Tokenizer, assets: &Assets, lang_id: Option, spk_id: Option, spk_emb: Option<&[f32]>, instruct: Option<&str>, mid_embeds: Option>>, ) -> Result { let mut embeds = Vec::new(); let text_ids = tokenizer.encode(text); // Check if text_ids is empty (e.g., only punctuation) if text_ids.is_empty() { return Err("Text is empty or contains only unsupported characters".to_string()); } // Check if text contains only punctuation and whitespace let text_trimmed: String = text.chars().filter(|c| !c.is_whitespace()).collect(); let only_punctuation = text_trimmed.chars().all(|c| { matches!( c, ',' | '。' | '、' | ';' | ':' | '?' | '!' | '"' | '\'' | '(' | ')' | '【' | '】' | '《' | '》' | '…' | '—' | '~' | ',' | '.' | '!' | '?' | ';' | ':' | '(' | ')' | '[' | ']' | '<' | '>' | '-' | '~' | '`' | '@' | '#' | '$' | '%' | '^' | '&' | '*' | '+' | '=' | '|' | '\\' | '/' | '{' | '}' | '_' ) }); if only_punctuation && !text_trimmed.is_empty() { return Err( "Text contains only punctuation marks, which may cause generation issues" .to_string(), ); } let result_spk_emb = spk_emb .map(|s| s.to_vec()) .unwrap_or_else(|| vec![0.0; 2048]); let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER); // 1. Instruct Block (User) if let Some(ins) = instruct { // <|im_start|>user\n let prefix = vec![151644, 872, 198]; for id in prefix { embeds.push(assets.get_text_embedding(id)); } let content_ids = tokenizer.encode(ins); for id in content_ids { embeds.push(assets.get_text_embedding(id as usize)); } // <|im_end|>\n let suffix = vec![151645, 198]; for id in suffix { embeds.push(assets.get_text_embedding(id)); } } // 2. Role Block (Assistant) // <|im_start|>assistant\n -> [151644, 77091, 198] for id in [151644, 77091, 198] { embeds.push(assets.get_text_embedding(id)); } // 3. Control Block if let Some(lid) = lang_id { // THINK, THINK_BOS, lang_id, THINK_EOS let ids = [THINK, THINK_BOS, lid, THINK_EOS]; for &id in &ids { let e = assets.get_codec_embedding(0, id as i32); let sum: Vec = marker_emb .iter() .zip(e.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } } else { // NOTHINK, THINK_BOS, THINK_EOS let ids = [NOTHINK, THINK_BOS, THINK_EOS]; for &id in &ids { let e = assets.get_codec_embedding(0, id as i32); let sum: Vec = marker_emb .iter() .zip(e.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } } // Speaker ID/Emb if let Some(sid) = spk_id { let e = assets.get_codec_embedding(0, sid as i32); let sum: Vec = marker_emb .iter() .zip(e.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } else if let Some(se) = spk_emb { let sum: Vec = marker_emb .iter() .zip(se.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(sum); } // 4. Mid Embeds if let Some(mids) = mid_embeds { embeds.extend(mids); } // 5. Task Text Block // BOS_TOKEN + PAD let pad_0 = assets.get_codec_embedding(0, PAD as i32); let bos_token_emb = assets.get_text_embedding(BOS_TOKEN); let bos_sum: Vec = bos_token_emb .iter() .zip(pad_0.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(bos_sum); for &id in &text_ids { let t_emb = assets.get_text_embedding(id as usize); let sum: Vec = t_emb.iter().zip(pad_0.iter()).map(|(a, b)| a + b).collect(); embeds.push(sum); } // EOS_TOKEN + PAD let eos_token_emb = assets.get_text_embedding(EOS_TOKEN); let eos_sum: Vec = eos_token_emb .iter() .zip(pad_0.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(eos_sum); // 6. Activation (BOS) // Python: assets.text_table[151671] + assets.emb_tables[0][p["BOS"]] let bos_emb = assets.get_codec_embedding(0, BOS as i32); let act_sum: Vec = marker_emb .iter() .zip(bos_emb.iter()) .map(|(a, b)| a + b) .collect(); embeds.push(act_sum); Ok(PromptData { embd: embeds, text_ids: text_ids.into_iter().collect(), spk_emb: result_spk_emb, }) } }