Files
Qwen3-TTS-Rust-Fixed/src/models/llama/mod.rs
T

1050 lines
36 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Llama.cpp FFI Bindings
use std::ffi::{c_char, c_float, c_void};
use std::os::raw::{c_int, c_uint};
use std::path::Path;
#[cfg(unix)]
fn ensure_runtime_library_link(runtime_path: &Path, lib_name: &str) {
let link_path = runtime_path.join(lib_name);
if link_path.exists() {
return;
}
if let Ok(output) = std::process::Command::new("gcc")
.arg(format!("-print-file-name={}", lib_name))
.output()
{
if output.status.success() {
let lib_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
let lib_path = std::path::PathBuf::from(lib_path);
if lib_path.exists() {
let _ = std::fs::create_dir_all(runtime_path);
let _ = std::os::unix::fs::symlink(&lib_path, &link_path);
}
}
}
}
#[cfg(not(unix))]
fn ensure_runtime_library_link(_runtime_path: &Path, _lib_name: &str) {}
#[repr(C)]
pub struct llama_model_params {
pub devices: *mut c_void,
pub tensor_buft_overrides: *mut c_void,
pub n_gpu_layers: c_int,
pub split_mode: c_int,
pub main_gpu: c_int,
pub tensor_split: *mut c_float,
pub progress_callback: *mut c_void,
pub progress_callback_user_data: *mut c_void,
pub kv_overrides: *mut c_void,
pub vocab_only: bool,
pub use_mmap: bool,
pub use_direct_io: bool,
pub use_mlock: bool,
pub check_tensors: bool,
pub use_extra_bufts: bool,
pub no_host: bool,
pub no_alloc: bool,
}
#[repr(C)]
pub struct llama_context_params {
pub n_ctx: c_uint,
pub n_batch: c_uint,
pub n_ubatch: c_uint,
pub n_seq_max: c_uint,
pub n_threads: c_int,
pub n_threads_batch: c_int,
pub rope_scaling_type: c_int,
pub pooling_type: c_int,
pub attention_type: c_int,
pub flash_attn_type: c_int,
pub rope_freq_base: c_float,
pub rope_freq_scale: c_float,
pub yarn_ext_factor: c_float,
pub yarn_attn_factor: c_float,
pub yarn_beta_fast: c_float,
pub yarn_beta_slow: c_float,
pub yarn_orig_ctx: c_uint,
pub defrag_thold: c_float,
pub cb_eval: *mut c_void,
pub cb_eval_user_data: *mut c_void,
pub type_k: c_int,
pub type_v: c_int,
pub abort_callback: *mut c_void,
pub abort_callback_data: *mut c_void,
pub embeddings: bool,
pub offload_kqv: bool,
pub no_perf: bool,
pub op_offload: bool,
pub swa_full: bool,
pub kv_unified: bool,
pub samplers: *mut c_void,
pub n_samplers: usize,
}
#[repr(C)]
#[derive(Copy, Clone)]
pub struct llama_batch {
pub n_tokens: c_int,
pub token: *mut c_int,
pub embd: *mut c_float,
pub pos: *mut c_int,
pub n_seq_id: *mut c_int,
pub seq_id: *mut *mut c_int,
pub logits: *mut i8,
}
pub type LlamaToken = c_int;
pub type LlamaModelPtr = *mut c_void;
pub type LlamaContextPtr = *mut c_void;
pub type LlamaSamplerPtr = *mut c_void;
pub type LlamaVocabPtr = *mut c_void;
type LlamaBackendInitFn = unsafe extern "C" fn();
type LlamaBackendFreeFn = unsafe extern "C" fn();
type LlamaModelDefaultParamsFn = unsafe extern "C" fn() -> llama_model_params;
type LlamaModelLoadFromFileFn =
unsafe extern "C" fn(*const c_char, llama_model_params) -> LlamaModelPtr;
type LlamaModelFreeFn = unsafe extern "C" fn(LlamaModelPtr);
type LlamaModelGetVocabFn = unsafe extern "C" fn(LlamaModelPtr) -> LlamaVocabPtr;
type LlamaModelNEmbdfn = unsafe extern "C" fn(LlamaModelPtr) -> c_int;
type LlamaModelNHeadFn = unsafe extern "C" fn(LlamaModelPtr) -> c_int;
type LlamaModelNLayerFn = unsafe extern "C" fn(LlamaModelPtr) -> c_int;
type LlamaModelNCtxFn = unsafe extern "C" fn() -> c_int;
type LlamaModelNVocabFn = unsafe extern "C" fn() -> c_int;
type LlamaVocabNTokensFn = unsafe extern "C" fn(LlamaVocabPtr) -> c_int;
type LlamaVocabEosFn = unsafe extern "C" fn(LlamaVocabPtr) -> LlamaToken;
type LlamaContextDefaultParamsFn = unsafe extern "C" fn() -> llama_context_params;
type LlamaInitFromModelFn =
unsafe extern "C" fn(LlamaModelPtr, llama_context_params) -> LlamaContextPtr;
type LlamaFreeFn = unsafe extern "C" fn(LlamaContextPtr);
type LlamaBatchInitFn = unsafe extern "C" fn(c_int, c_int, c_int) -> llama_batch;
type LlamaBatchFreeFn = unsafe extern "C" fn(llama_batch);
type LlamaDecodeFn = unsafe extern "C" fn(LlamaContextPtr, llama_batch) -> c_int;
type LlamaGetEmbeddingsFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_float;
type LlamaGetLogitsFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_float;
type LlamaGetMemoryFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_void;
type LlamaMemoryClearFn = unsafe extern "C" fn(*mut c_void, bool);
type LlamaMemorySeqRmFn = unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int) -> bool;
type LlamaMemorySeqPosMaxFn = unsafe extern "C" fn(*mut c_void, c_int) -> c_int;
type LlamaSamplerInitTempFn = unsafe extern "C" fn(c_float) -> LlamaSamplerPtr;
type LlamaSamplerSampleFn =
unsafe extern "C" fn(LlamaSamplerPtr, LlamaContextPtr, c_int) -> LlamaToken;
type LlamaSamplerFreeFn = unsafe extern "C" fn(LlamaSamplerPtr);
type GgmlBackendLoadAllFn = unsafe extern "C" fn();
pub struct LlamaFFI {
pub llama_backend_init: LlamaBackendInitFn,
pub llama_backend_free: LlamaBackendFreeFn,
pub llama_model_default_params: LlamaModelDefaultParamsFn,
pub llama_model_load_from_file: LlamaModelLoadFromFileFn,
pub llama_model_free: LlamaModelFreeFn,
pub llama_model_get_vocab: LlamaModelGetVocabFn,
pub llama_model_n_embd: LlamaModelNEmbdfn,
pub llama_model_n_head: LlamaModelNHeadFn,
pub llama_model_n_layer: LlamaModelNLayerFn,
pub llama_model_n_ctx: LlamaModelNCtxFn,
pub llama_model_n_vocab: LlamaModelNVocabFn,
pub llama_vocab_n_tokens: LlamaVocabNTokensFn,
pub llama_vocab_eos: LlamaVocabEosFn,
pub llama_context_default_params: LlamaContextDefaultParamsFn,
pub llama_init_from_model: LlamaInitFromModelFn,
pub llama_free: LlamaFreeFn,
pub llama_batch_init: LlamaBatchInitFn,
pub llama_batch_free: LlamaBatchFreeFn,
pub llama_decode: LlamaDecodeFn,
pub llama_get_embeddings: LlamaGetEmbeddingsFn,
pub llama_get_logits: LlamaGetLogitsFn,
pub llama_get_memory: LlamaGetMemoryFn,
pub llama_memory_clear: LlamaMemoryClearFn,
pub llama_memory_seq_rm: LlamaMemorySeqRmFn,
pub llama_memory_seq_pos_max: LlamaMemorySeqPosMaxFn,
pub llama_sampler_init_temp: LlamaSamplerInitTempFn,
pub llama_sampler_sample: LlamaSamplerSampleFn,
pub llama_sampler_free: LlamaSamplerFreeFn,
pub ggml_backend_load_all: GgmlBackendLoadAllFn,
}
static FFI: std::sync::OnceLock<LlamaFFI> = std::sync::OnceLock::new();
pub fn get_ffi() -> &'static LlamaFFI {
FFI.get_or_init(|| {
unsafe {
let runtime_path = std::env::current_dir().unwrap().join("runtime");
let runtime_path_str = runtime_path.to_string_lossy();
let path_key = if cfg!(target_os = "windows") {
"PATH"
} else if cfg!(target_os = "macos") {
"DYLD_LIBRARY_PATH"
} else {
"LD_LIBRARY_PATH"
};
let path_var = std::env::var_os(path_key).unwrap_or_default();
let mut paths: Vec<_> = std::env::split_paths(&path_var).collect();
if !paths.contains(&runtime_path) {
paths.insert(0, runtime_path.clone());
let new_path = std::env::join_paths(paths).unwrap();
std::env::set_var(path_key, new_path);
println!(" [System] Added to {}: {}", path_key, runtime_path_str);
}
#[cfg(target_os = "windows")]
{
use std::os::windows::ffi::OsStrExt;
let path_wide = std::ffi::OsStr::new(&runtime_path)
.encode_wide()
.chain(std::iter::once(0))
.collect::<Vec<u16>>();
winapi::um::winbase::SetDllDirectoryW(path_wide.as_ptr());
println!(" [System] SetDllDirectoryW set to: {}", runtime_path_str);
}
let (ggml_name, llama_name, libomp_name) = if cfg!(target_os = "windows") {
("ggml.dll", "llama.dll", Some("libomp140.x86_64.dll"))
} else if cfg!(target_os = "macos") {
("libggml.dylib", "libllama.dylib", None)
} else {
("libggml.so", "libllama.so", None)
};
if let Some(omp_name) = libomp_name {
let libomp_path = runtime_path.join(omp_name);
let _libomp = libloading::Library::new(&libomp_path).ok();
}
ensure_runtime_library_link(&runtime_path, "libgomp.so.1");
let ggml_path = runtime_path.join(ggml_name);
let ggml_lib = libloading::Library::new(&ggml_path)
.map_err(|e| {
eprintln!(
"WARNING: Failed to load {} from {:?}: {}",
ggml_name, ggml_path, e
)
})
.ok();
let cpu_backend_name = if cfg!(target_arch = "x86_64") {
Some("libggml-cpu-x64.so")
} else if cfg!(target_arch = "aarch64") {
Some("libggml-cpu-arm64.so")
} else {
None
};
let cpu_backend_lib = cpu_backend_name
.and_then(|name| {
let cpu_backend_path = runtime_path.join(name);
match libloading::Library::new(&cpu_backend_path) {
Ok(lib) => Some(lib),
Err(e) => {
eprintln!(
"WARNING: Failed to load CPU backend {} from {:?}: {}",
name, cpu_backend_path, e
);
None
}
}
});
#[cfg(target_os = "linux")]
{
if let Some(name) = cpu_backend_name {
let backend_path = runtime_path.join(name);
std::env::set_var("GGML_BACKEND_PATH", &backend_path);
println!(" [System] GGML_BACKEND_PATH set to: {}", backend_path.to_string_lossy());
}
}
let llama_path = runtime_path.join(llama_name);
let lib = libloading::Library::new(&llama_path)
.unwrap_or_else(|_| panic!("Failed to load {}. Please ensure it is in the runtime/ directory.", llama_name));
unsafe extern "C" fn dummy_fn() {}
let load_all_fn: GgmlBackendLoadAllFn = if let Some(ref glib) = ggml_lib {
match glib.get::<GgmlBackendLoadAllFn>(b"ggml_backend_load_all") {
Ok(symbol) => *symbol,
Err(_) => dummy_fn,
}
} else {
lib.get::<GgmlBackendLoadAllFn>(b"ggml_backend_load_all")
.map(|s| *s)
.unwrap_or(dummy_fn)
};
let vulkan_path = runtime_path.join("libggml-vulkan.so");
match libloading::Library::new(&vulkan_path) {
Ok(_) => println!(" [System] Verified: libggml-vulkan.so is loadable."),
Err(e) => eprintln!(" [System] WARNING: libggml-vulkan.so is NOT loadable: {}. This usually means missing Vulkan drivers or runtime.", e),
}
let ffi = LlamaFFI {
llama_backend_init: *lib.get(b"llama_backend_init").expect("llama_backend_init"),
llama_backend_free: *lib.get(b"llama_backend_free").expect("llama_backend_free"),
llama_model_default_params: *lib
.get(b"llama_model_default_params")
.expect("llama_model_default_params"),
llama_model_load_from_file: *lib
.get(b"llama_model_load_from_file")
.expect("llama_model_load_from_file"),
llama_model_free: *lib.get(b"llama_model_free").expect("llama_model_free"),
llama_model_get_vocab: *lib
.get(b"llama_model_get_vocab")
.expect("llama_model_get_vocab"),
llama_model_n_embd: *lib.get(b"llama_model_n_embd").expect("llama_model_n_embd"),
llama_model_n_head: *lib.get(b"llama_model_n_head").expect("llama_model_n_head"),
llama_model_n_layer: *lib
.get(b"llama_model_n_layer")
.expect("llama_model_n_layer"),
llama_model_n_ctx: *lib.get(b"llama_n_ctx").expect("llama_n_ctx"),
llama_model_n_vocab: *lib.get(b"llama_n_vocab").expect("llama_n_vocab"),
llama_vocab_n_tokens: *lib
.get(b"llama_vocab_n_tokens")
.expect("llama_vocab_n_tokens"),
llama_vocab_eos: *lib.get(b"llama_vocab_eos").expect("llama_vocab_eos"),
llama_context_default_params: *lib
.get(b"llama_context_default_params")
.expect("llama_context_default_params"),
llama_init_from_model: *lib
.get(b"llama_init_from_model")
.expect("llama_init_from_model"),
llama_free: *lib.get(b"llama_free").expect("llama_free"),
llama_batch_init: *lib.get(b"llama_batch_init").expect("llama_batch_init"),
llama_batch_free: *lib.get(b"llama_batch_free").expect("llama_batch_free"),
llama_decode: *lib.get(b"llama_decode").expect("llama_decode"),
llama_get_embeddings: *lib
.get(b"llama_get_embeddings")
.expect("llama_get_embeddings"),
llama_get_logits: *lib.get(b"llama_get_logits").expect("llama_get_logits"),
llama_get_memory: *lib.get(b"llama_get_memory").expect("llama_get_memory"),
llama_memory_clear: *lib.get(b"llama_memory_clear").expect("llama_memory_clear"),
llama_memory_seq_rm: *lib
.get(b"llama_memory_seq_rm")
.expect("llama_memory_seq_rm"),
llama_memory_seq_pos_max: *lib
.get(b"llama_memory_seq_pos_max")
.expect("llama_memory_seq_pos_max"),
llama_sampler_init_temp: *lib
.get(b"llama_sampler_init_temp")
.expect("llama_sampler_init_temp"),
llama_sampler_sample: *lib
.get(b"llama_sampler_sample")
.expect("llama_sampler_sample"),
llama_sampler_free: *lib.get(b"llama_sampler_free").expect("llama_sampler_free"),
ggml_backend_load_all: load_all_fn,
};
let original_cwd = std::env::current_dir().ok();
if original_cwd.is_some() {
let _ = std::env::set_current_dir(&runtime_path);
}
(ffi.ggml_backend_load_all)();
(ffi.llama_backend_init)();
if let Some(cwd) = original_cwd {
let _ = std::env::set_current_dir(cwd);
}
std::mem::forget(lib);
if let Some(glib) = ggml_lib {
std::mem::forget(glib);
}
if let Some(cpu_backend_lib) = cpu_backend_lib {
std::mem::forget(cpu_backend_lib);
}
ffi
}
})
}
pub fn cleanup() {
if let Some(ffi) = FFI.get() {
unsafe {
(ffi.llama_backend_free)();
}
}
}
pub struct LlamaModel {
pub ptr: LlamaModelPtr,
pub vocab: LlamaVocabPtr,
pub n_embd: usize,
pub n_head: usize,
pub n_layer: usize,
pub n_ctx: usize,
pub n_vocab: usize,
pub eos_token: LlamaToken,
}
unsafe impl Send for LlamaModel {}
unsafe impl Sync for LlamaModel {}
impl LlamaModel {
pub fn load(path: &Path, n_gpu_layers: i32) -> Result<Self, String> {
let ffi = get_ffi();
unsafe {
let mut params = (ffi.llama_model_default_params)();
params.n_gpu_layers = n_gpu_layers;
let c_path =
std::ffi::CString::new(path.to_str().unwrap()).map_err(|e| e.to_string())?;
let ptr = (ffi.llama_model_load_from_file)(c_path.as_ptr(), params);
if ptr.is_null() {
return Err("Failed to load model".to_string());
}
let vocab = (ffi.llama_model_get_vocab)(ptr);
let n_vocab = (ffi.llama_vocab_n_tokens)(vocab) as usize;
let n_embd = (ffi.llama_model_n_embd)(ptr) as usize;
let n_head = (ffi.llama_model_n_head)(ptr) as usize;
let n_layer = (ffi.llama_model_n_layer)(ptr) as usize;
let eos_token = (ffi.llama_vocab_eos)(vocab);
Ok(Self {
ptr,
vocab,
n_embd,
n_head,
n_layer,
n_ctx: 32768,
n_vocab,
eos_token,
})
}
}
}
impl Drop for LlamaModel {
fn drop(&mut self) {
unsafe {
if let Some(ffi) = FFI.get() {
if !self.ptr.is_null() {
(ffi.llama_model_free)(self.ptr);
}
}
}
}
}
impl Clone for LlamaModel {
fn clone(&self) -> Self {
Self {
ptr: self.ptr,
vocab: self.vocab,
n_embd: self.n_embd,
n_head: self.n_head,
n_layer: self.n_layer,
n_ctx: self.n_ctx,
n_vocab: self.n_vocab,
eos_token: self.eos_token,
}
}
}
pub struct LlamaContext {
pub ptr: LlamaContextPtr,
pub model: std::mem::ManuallyDrop<LlamaModel>,
}
unsafe impl Send for LlamaContext {}
unsafe impl Sync for LlamaContext {}
impl LlamaContext {
pub fn new(
model: &LlamaModel,
n_ctx: u32,
n_batch: u32,
embeddings: i32,
n_threads: i32,
) -> Result<Self, String> {
let ffi = get_ffi();
unsafe {
let mut params = (ffi.llama_context_default_params)();
params.n_ctx = n_ctx;
params.n_batch = n_batch;
params.n_ubatch = 512;
params.n_seq_max = 1;
params.embeddings = embeddings != 0;
params.flash_attn_type = 1;
params.offload_kqv = true;
params.no_perf = true;
if n_threads > 0 {
params.n_threads = n_threads;
} else {
let cpu_count = std::thread::available_parallelism()
.map(|p| p.get() as c_int)
.unwrap_or(4);
params.n_threads = (cpu_count / 2).min(4);
}
let ptr = (ffi.llama_init_from_model)(model.ptr, params);
if ptr.is_null() {
return Err("Failed to create context".to_string());
}
Ok(Self {
ptr,
model: std::mem::ManuallyDrop::new(model.clone()),
})
}
}
pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), String> {
let ffi = get_ffi();
unsafe {
let result = (ffi.llama_decode)(self.ptr, batch.batch);
if result != 0 {
return Err(format!("llama_decode failed: {}", result));
}
Ok(())
}
}
pub fn get_embeddings(&self) -> &[f32] {
let ffi = get_ffi();
unsafe {
std::slice::from_raw_parts((ffi.llama_get_embeddings)(self.ptr), self.model.n_embd)
}
}
pub fn get_embedding_at(&self, batch_index: usize) -> &[f32] {
let ffi = get_ffi();
unsafe {
let base_ptr = (ffi.llama_get_embeddings)(self.ptr);
let offset = batch_index * self.model.n_embd;
std::slice::from_raw_parts(base_ptr.add(offset), self.model.n_embd)
}
}
pub fn get_logits(&self) -> &[f32] {
let ffi = get_ffi();
unsafe { std::slice::from_raw_parts((ffi.llama_get_logits)(self.ptr), self.model.n_vocab) }
}
pub fn get_logits_ith(&self, i: usize) -> &[f32] {
let ffi = get_ffi();
unsafe {
let base_ptr = (ffi.llama_get_logits)(self.ptr);
std::slice::from_raw_parts(base_ptr.add(i * self.model.n_vocab), self.model.n_vocab)
}
}
pub fn clear_kv_cache(&self) -> bool {
let ffi = get_ffi();
unsafe {
let mem = (ffi.llama_get_memory)(self.ptr);
let result = (ffi.llama_memory_seq_rm)(mem, -1, 0, -1);
if !result {
eprintln!("WARNING: clear_kv_cache failed for ctx {:?}", self.ptr);
}
result
}
}
pub fn debug_seq_pos_max(&self, seq_id: c_int) -> c_int {
let ffi = get_ffi();
unsafe {
let mem = (ffi.llama_get_memory)(self.ptr);
let result = (ffi.llama_memory_seq_pos_max)(mem, seq_id);
eprintln!(
"DEBUG: [debug_seq_pos_max] mem={:?}, seq_id={}, result={}",
mem, seq_id, result
);
result
}
}
}
impl Drop for LlamaContext {
fn drop(&mut self) {
unsafe {
if let Some(ffi) = FFI.get() {
if !self.ptr.is_null() {
(ffi.llama_free)(self.ptr);
}
}
}
}
}
pub struct LlamaBatch {
batch: llama_batch,
n_tokens_max: usize,
n_embd: usize,
_n_pos_per_embd: usize,
_embd_buffer: Vec<c_float>,
_pos_buffer: Vec<c_int>,
_seq_id_buffers: Vec<Vec<c_int>>,
_n_seq_id_buffer: Vec<c_int>,
_logits_buffer: Vec<i8>,
}
impl LlamaBatch {
pub fn new(
n_tokens_max: usize,
n_embd: usize,
n_seq_max: usize,
n_pos_per_embd: usize,
) -> Self {
let ffi = get_ffi();
unsafe {
let batch =
(ffi.llama_batch_init)(n_tokens_max as c_int, n_embd as c_int, n_seq_max as c_int);
Self {
batch,
n_tokens_max,
n_embd,
_n_pos_per_embd: n_pos_per_embd,
_embd_buffer: Vec::new(),
_pos_buffer: Vec::new(),
_seq_id_buffers: Vec::new(),
_n_seq_id_buffer: Vec::new(),
_logits_buffer: Vec::new(),
}
}
}
pub fn set_embd(&mut self, prompt_embeds: &[f32], pos_arr: &[i32], seq_id: i32) {
let n_tokens = prompt_embeds.len() / self.n_embd;
unsafe {
std::ptr::copy_nonoverlapping(
prompt_embeds.as_ptr(),
self.batch.embd,
prompt_embeds.len(),
);
let max_pos = self.n_tokens_max;
if pos_arr.len() > max_pos {
eprintln!(
"WARNING: pos_arr length {} exceeds batch capacity {}. Truncating!",
pos_arr.len(),
max_pos
);
}
std::ptr::copy_nonoverlapping(
pos_arr.as_ptr(),
self.batch.pos,
pos_arr.len().min(max_pos),
);
}
for i in 0..n_tokens {
unsafe {
*self.batch.n_seq_id.add(i) = 1;
let seq_ids = *self.batch.seq_id.add(i);
*seq_ids.add(0) = seq_id;
*self.batch.logits.add(i) = if i == n_tokens - 1 { 1 } else { 0 };
}
}
self.batch.n_tokens = n_tokens as c_int;
}
pub fn clear(&mut self) {
self.batch.n_tokens = 0;
}
pub fn n_tokens(&self) -> usize {
self.batch.n_tokens as usize
}
pub fn batch_ptr(&mut self) -> *mut llama_batch {
&mut self.batch
}
}
pub struct LlamaSampler {
_ptr: LlamaSamplerPtr,
n_vocab: usize,
_neg_inf: f32,
temperature: f32,
top_k: usize,
top_p: f32,
min_p: f32,
repeat_penalty: f32,
frequency_penalty: f32,
presence_penalty: f32,
penalty_last_n: usize,
token_history: std::cell::RefCell<Vec<LlamaToken>>,
rng: std::cell::RefCell<rand::rngs::StdRng>,
}
impl LlamaSampler {
pub fn new(n_vocab: usize, temperature: f32, top_k: i32, top_p: f32, seed: u64) -> Self {
use rand::SeedableRng;
Self {
_ptr: std::ptr::null_mut(),
n_vocab,
_neg_inf: -1e9,
temperature,
top_k: top_k as usize,
top_p,
min_p: 0.0,
repeat_penalty: 1.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
penalty_last_n: 64,
token_history: std::cell::RefCell::new(Vec::new()),
rng: std::cell::RefCell::new(rand::rngs::StdRng::seed_from_u64(seed)),
}
}
pub fn greedy(n_vocab: usize) -> Self {
use rand::SeedableRng;
Self {
_ptr: std::ptr::null_mut(),
n_vocab,
_neg_inf: -1e9,
temperature: 0.0,
top_k: 0,
top_p: 1.0,
min_p: 0.0,
repeat_penalty: 1.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
penalty_last_n: 64,
token_history: std::cell::RefCell::new(Vec::new()),
rng: std::cell::RefCell::new(rand::rngs::StdRng::seed_from_u64(42)),
}
}
pub fn with_penalties(
mut self,
min_p: f32,
repeat_penalty: f32,
frequency_penalty: f32,
presence_penalty: f32,
penalty_last_n: usize,
) -> Self {
self.min_p = min_p;
self.repeat_penalty = repeat_penalty;
self.frequency_penalty = frequency_penalty;
self.presence_penalty = presence_penalty;
self.penalty_last_n = penalty_last_n;
self
}
pub fn accept(&self, token: LlamaToken) {
let mut history = self.token_history.borrow_mut();
history.push(token);
if history.len() > self.penalty_last_n {
let excess = history.len() - self.penalty_last_n;
history.drain(0..excess);
}
}
fn apply_penalties(&self, logits: &mut [f32]) {
if self.repeat_penalty == 1.0
&& self.frequency_penalty == 0.0
&& self.presence_penalty == 0.0
{
return;
}
let history = self.token_history.borrow();
let mut token_counts: std::collections::HashMap<LlamaToken, usize> =
std::collections::HashMap::new();
for &token in history.iter() {
*token_counts.entry(token).or_insert(0) += 1;
}
// 惩罚豁免名单:EOS(2150)、PAD(2148)、BOS(2149)、TTS_EOS(151673)
let penalty_exempt: std::collections::HashSet<LlamaToken> =
[2150, 2148, 2149, 151673].iter().cloned().collect();
for (&token, &count) in token_counts.iter() {
// 跳过豁免名单中的 token
if penalty_exempt.contains(&token) {
continue;
}
let idx = token as usize;
if idx < logits.len() {
if self.repeat_penalty != 1.0 {
if logits[idx] > 0.0 {
logits[idx] /= self.repeat_penalty;
} else {
logits[idx] *= self.repeat_penalty;
}
}
logits[idx] -= self.frequency_penalty * count as f32;
logits[idx] -= self.presence_penalty;
}
}
}
fn apply_min_p(&self, probs: &mut Vec<(usize, f32)>) {
if self.min_p <= 0.0 || probs.is_empty() {
return;
}
let max_prob = probs.iter().map(|(_, p)| *p).fold(0.0, f32::max);
let threshold = max_prob * self.min_p;
probs.retain(|(_, p)| *p >= threshold);
}
pub fn sample(
&self,
ctx: &LlamaContext,
idx: i32,
limit_start: Option<usize>,
limit_end: Option<usize>,
) -> LlamaToken {
self.sample_with_allow(ctx, idx, limit_start, limit_end, None)
}
pub fn sample_with_allow(
&self,
ctx: &LlamaContext,
idx: i32,
limit_start: Option<usize>,
limit_end: Option<usize>,
allow_tokens: Option<&[LlamaToken]>,
) -> LlamaToken {
let ffi = get_ffi();
unsafe {
let offset = if idx >= 0 { idx as usize } else { 0 };
let base_ptr = (ffi.llama_get_logits)(ctx.ptr);
let logits_ptr = base_ptr.add(offset * self.n_vocab);
let logits_raw = std::slice::from_raw_parts(logits_ptr, self.n_vocab);
let mut logits: Vec<f32> = logits_raw.to_vec();
let start = limit_start.unwrap_or(0);
let end = limit_end.unwrap_or(self.n_vocab).min(self.n_vocab);
let mut mask = vec![true; self.n_vocab];
for m in mask.iter_mut().take(end).skip(start) {
*m = false;
}
if let Some(allow) = allow_tokens {
for &token in allow {
let t = token as usize;
if t < self.n_vocab {
mask[t] = false;
}
}
}
for (i, &m) in mask.iter().enumerate() {
if m {
logits[i] = self._neg_inf;
}
}
self.apply_penalties(&mut logits);
if self.temperature <= 0.0 {
let mut max_val = f32::NEG_INFINITY;
let mut max_idx = 0;
for (i, &val) in logits.iter().enumerate() {
if val > max_val {
max_val = val;
max_idx = i;
}
}
self.accept(max_idx as LlamaToken);
return max_idx as LlamaToken;
}
let mut candidates: Vec<(usize, f32)> = logits
.iter()
.enumerate()
.filter(|(_, &v)| v > f32::NEG_INFINITY / 2.0)
.map(|(i, &v)| (i, v))
.collect();
candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
if self.top_k > 0 && self.top_k < candidates.len() {
candidates.truncate(self.top_k);
}
let max_logit = candidates.first().map(|(_, l)| *l).unwrap_or(0.0);
let mut probs: Vec<(usize, f32)> = candidates
.iter()
.map(|(idx, logit)| {
let scaled = (*logit - max_logit) / self.temperature;
(*idx, scaled.exp())
})
.collect();
let sum: f32 = probs.iter().map(|(_, p)| *p).sum();
if sum > 0.0 {
for (_, p) in probs.iter_mut() {
*p /= sum;
}
}
self.apply_min_p(&mut probs);
if self.top_p < 1.0 {
let mut cumsum = 0.0;
let mut cutoff_idx = probs.len();
for (i, (_, p)) in probs.iter().enumerate() {
cumsum += p;
if cumsum >= self.top_p {
cutoff_idx = i + 1;
break;
}
}
probs.truncate(cutoff_idx);
let new_sum: f32 = probs.iter().map(|(_, p)| *p).sum();
if new_sum > 0.0 {
for (_, p) in probs.iter_mut() {
*p /= new_sum;
}
}
}
use rand::Rng;
let r: f32 = self.rng.borrow_mut().gen();
let mut cumsum = 0.0;
for (idx, p) in probs.iter() {
cumsum += p;
if r < cumsum {
self.accept(*idx as LlamaToken);
return *idx as LlamaToken;
}
}
let result = probs
.first()
.map(|(idx, _)| *idx as LlamaToken)
.unwrap_or(start as LlamaToken);
self.accept(result);
result
}
}
/// 采样时对静音 token 施加惩罚
/// silent_penalty: 静音惩罚系数(0-1),降低静音 token 的概率
/// silent_threshold: 静音 token 的阈值(code_0 < silent_threshold 被视为静音)
pub fn sample_with_silent_penalty(
&self,
ctx: &LlamaContext,
idx: i32,
limit_start: Option<usize>,
limit_end: Option<usize>,
allow_tokens: Option<&[LlamaToken]>,
silent_penalty: f32,
silent_threshold: usize,
) -> LlamaToken {
let ffi = get_ffi();
unsafe {
let offset = if idx >= 0 { idx as usize } else { 0 };
let base_ptr = (ffi.llama_get_logits)(ctx.ptr);
let logits_ptr = base_ptr.add(offset * self.n_vocab);
let logits_raw = std::slice::from_raw_parts(logits_ptr, self.n_vocab);
let mut logits: Vec<f32> = logits_raw.to_vec();
let start = limit_start.unwrap_or(0);
let end = limit_end.unwrap_or(self.n_vocab).min(self.n_vocab);
// 创建掩码
let mut mask = vec![true; self.n_vocab];
for m in mask.iter_mut().take(end).skip(start) {
*m = false;
}
if let Some(allow) = allow_tokens {
for &token in allow {
let t = token as usize;
if t < self.n_vocab {
mask[t] = false;
}
}
}
for (i, &m) in mask.iter().enumerate() {
if m {
logits[i] = self._neg_inf;
}
}
// 对静音 token 施加惩罚(降低 logits)
if silent_penalty > 0.0 {
for i in 0..silent_threshold.min(end) {
if !mask[i] {
logits[i] -= silent_penalty;
}
}
}
self.apply_penalties(&mut logits);
if self.temperature <= 0.0 {
let mut max_val = f32::NEG_INFINITY;
let mut max_idx = 0;
for (i, &val) in logits.iter().enumerate() {
if val > max_val {
max_val = val;
max_idx = i;
}
}
self.accept(max_idx as LlamaToken);
return max_idx as LlamaToken;
}
let mut candidates: Vec<(usize, f32)> = logits
.iter()
.enumerate()
.filter(|(_, &v)| v > f32::NEG_INFINITY / 2.0)
.map(|(i, &v)| (i, v))
.collect();
candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
if self.top_k > 0 && self.top_k < candidates.len() {
candidates.truncate(self.top_k);
}
let max_logit = candidates.first().map(|(_, l)| *l).unwrap_or(0.0);
let mut probs: Vec<(usize, f32)> = candidates
.iter()
.map(|(idx, logit)| {
let scaled = (*logit - max_logit) / self.temperature;
(*idx, scaled.exp())
})
.collect();
let sum: f32 = probs.iter().map(|(_, p)| *p).sum();
if sum > 0.0 {
for (_, p) in probs.iter_mut() {
*p /= sum;
}
}
self.apply_min_p(&mut probs);
if self.top_p < 1.0 {
let mut cumsum = 0.0;
let mut cutoff_idx = probs.len();
for (i, (_, p)) in probs.iter().enumerate() {
cumsum += p;
if cumsum > self.top_p {
cutoff_idx = i + 1;
break;
}
}
probs.truncate(cutoff_idx);
}
use rand::Rng;
let r: f32 = self.rng.borrow_mut().gen();
let mut cumsum = 0.0;
for (idx, p) in probs.iter() {
cumsum += p;
if r < cumsum {
self.accept(*idx as LlamaToken);
return *idx as LlamaToken;
}
}
let result = probs
.first()
.map(|(idx, _)| *idx as LlamaToken)
.unwrap_or(start as LlamaToken);
self.accept(result);
result
}
}
}
impl Drop for LlamaSampler {
fn drop(&mut self) {}
}