//! Llama.cpp FFI Bindings use std::ffi::{c_char, c_float, c_void}; use std::os::raw::{c_int, c_uint}; use std::path::Path; #[cfg(unix)] fn ensure_runtime_library_link(runtime_path: &Path, lib_name: &str) { let link_path = runtime_path.join(lib_name); if link_path.exists() { return; } if let Ok(output) = std::process::Command::new("gcc") .arg(format!("-print-file-name={}", lib_name)) .output() { if output.status.success() { let lib_path = String::from_utf8_lossy(&output.stdout).trim().to_string(); let lib_path = std::path::PathBuf::from(lib_path); if lib_path.exists() { let _ = std::fs::create_dir_all(runtime_path); let _ = std::os::unix::fs::symlink(&lib_path, &link_path); } } } } #[cfg(not(unix))] fn ensure_runtime_library_link(_runtime_path: &Path, _lib_name: &str) {} #[repr(C)] pub struct llama_model_params { pub devices: *mut c_void, pub tensor_buft_overrides: *mut c_void, pub n_gpu_layers: c_int, pub split_mode: c_int, pub main_gpu: c_int, pub tensor_split: *mut c_float, pub progress_callback: *mut c_void, pub progress_callback_user_data: *mut c_void, pub kv_overrides: *mut c_void, pub vocab_only: bool, pub use_mmap: bool, pub use_direct_io: bool, pub use_mlock: bool, pub check_tensors: bool, pub use_extra_bufts: bool, pub no_host: bool, pub no_alloc: bool, } #[repr(C)] pub struct llama_context_params { pub n_ctx: c_uint, pub n_batch: c_uint, pub n_ubatch: c_uint, pub n_seq_max: c_uint, pub n_threads: c_int, pub n_threads_batch: c_int, pub rope_scaling_type: c_int, pub pooling_type: c_int, pub attention_type: c_int, pub flash_attn_type: c_int, pub rope_freq_base: c_float, pub rope_freq_scale: c_float, pub yarn_ext_factor: c_float, pub yarn_attn_factor: c_float, pub yarn_beta_fast: c_float, pub yarn_beta_slow: c_float, pub yarn_orig_ctx: c_uint, pub defrag_thold: c_float, pub cb_eval: *mut c_void, pub cb_eval_user_data: *mut c_void, pub type_k: c_int, pub type_v: c_int, pub abort_callback: *mut c_void, pub abort_callback_data: *mut c_void, pub embeddings: bool, pub offload_kqv: bool, pub no_perf: bool, pub op_offload: bool, pub swa_full: bool, pub kv_unified: bool, pub samplers: *mut c_void, pub n_samplers: usize, } #[repr(C)] #[derive(Copy, Clone)] pub struct llama_batch { pub n_tokens: c_int, pub token: *mut c_int, pub embd: *mut c_float, pub pos: *mut c_int, pub n_seq_id: *mut c_int, pub seq_id: *mut *mut c_int, pub logits: *mut i8, } pub type LlamaToken = c_int; pub type LlamaModelPtr = *mut c_void; pub type LlamaContextPtr = *mut c_void; pub type LlamaSamplerPtr = *mut c_void; pub type LlamaVocabPtr = *mut c_void; type LlamaBackendInitFn = unsafe extern "C" fn(); type LlamaBackendFreeFn = unsafe extern "C" fn(); type LlamaModelDefaultParamsFn = unsafe extern "C" fn() -> llama_model_params; type LlamaModelLoadFromFileFn = unsafe extern "C" fn(*const c_char, llama_model_params) -> LlamaModelPtr; type LlamaModelFreeFn = unsafe extern "C" fn(LlamaModelPtr); type LlamaModelGetVocabFn = unsafe extern "C" fn(LlamaModelPtr) -> LlamaVocabPtr; type LlamaModelNEmbdfn = unsafe extern "C" fn(LlamaModelPtr) -> c_int; type LlamaModelNHeadFn = unsafe extern "C" fn(LlamaModelPtr) -> c_int; type LlamaModelNLayerFn = unsafe extern "C" fn(LlamaModelPtr) -> c_int; type LlamaModelNCtxFn = unsafe extern "C" fn() -> c_int; type LlamaModelNVocabFn = unsafe extern "C" fn() -> c_int; type LlamaVocabNTokensFn = unsafe extern "C" fn(LlamaVocabPtr) -> c_int; type LlamaVocabEosFn = unsafe extern "C" fn(LlamaVocabPtr) -> LlamaToken; type LlamaContextDefaultParamsFn = unsafe extern "C" fn() -> llama_context_params; type LlamaInitFromModelFn = unsafe extern "C" fn(LlamaModelPtr, llama_context_params) -> LlamaContextPtr; type LlamaFreeFn = unsafe extern "C" fn(LlamaContextPtr); type LlamaBatchInitFn = unsafe extern "C" fn(c_int, c_int, c_int) -> llama_batch; type LlamaBatchFreeFn = unsafe extern "C" fn(llama_batch); type LlamaDecodeFn = unsafe extern "C" fn(LlamaContextPtr, llama_batch) -> c_int; type LlamaGetEmbeddingsFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_float; type LlamaGetLogitsFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_float; type LlamaGetMemoryFn = unsafe extern "C" fn(LlamaContextPtr) -> *mut c_void; type LlamaMemoryClearFn = unsafe extern "C" fn(*mut c_void, bool); type LlamaMemorySeqRmFn = unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int) -> bool; type LlamaMemorySeqPosMaxFn = unsafe extern "C" fn(*mut c_void, c_int) -> c_int; type LlamaSamplerInitTempFn = unsafe extern "C" fn(c_float) -> LlamaSamplerPtr; type LlamaSamplerSampleFn = unsafe extern "C" fn(LlamaSamplerPtr, LlamaContextPtr, c_int) -> LlamaToken; type LlamaSamplerFreeFn = unsafe extern "C" fn(LlamaSamplerPtr); type GgmlBackendLoadAllFn = unsafe extern "C" fn(); pub struct LlamaFFI { pub llama_backend_init: LlamaBackendInitFn, pub llama_backend_free: LlamaBackendFreeFn, pub llama_model_default_params: LlamaModelDefaultParamsFn, pub llama_model_load_from_file: LlamaModelLoadFromFileFn, pub llama_model_free: LlamaModelFreeFn, pub llama_model_get_vocab: LlamaModelGetVocabFn, pub llama_model_n_embd: LlamaModelNEmbdfn, pub llama_model_n_head: LlamaModelNHeadFn, pub llama_model_n_layer: LlamaModelNLayerFn, pub llama_model_n_ctx: LlamaModelNCtxFn, pub llama_model_n_vocab: LlamaModelNVocabFn, pub llama_vocab_n_tokens: LlamaVocabNTokensFn, pub llama_vocab_eos: LlamaVocabEosFn, pub llama_context_default_params: LlamaContextDefaultParamsFn, pub llama_init_from_model: LlamaInitFromModelFn, pub llama_free: LlamaFreeFn, pub llama_batch_init: LlamaBatchInitFn, pub llama_batch_free: LlamaBatchFreeFn, pub llama_decode: LlamaDecodeFn, pub llama_get_embeddings: LlamaGetEmbeddingsFn, pub llama_get_logits: LlamaGetLogitsFn, pub llama_get_memory: LlamaGetMemoryFn, pub llama_memory_clear: LlamaMemoryClearFn, pub llama_memory_seq_rm: LlamaMemorySeqRmFn, pub llama_memory_seq_pos_max: LlamaMemorySeqPosMaxFn, pub llama_sampler_init_temp: LlamaSamplerInitTempFn, pub llama_sampler_sample: LlamaSamplerSampleFn, pub llama_sampler_free: LlamaSamplerFreeFn, pub ggml_backend_load_all: GgmlBackendLoadAllFn, } static FFI: std::sync::OnceLock = std::sync::OnceLock::new(); pub fn get_ffi() -> &'static LlamaFFI { FFI.get_or_init(|| { unsafe { let runtime_path = std::env::current_dir().unwrap().join("runtime"); let runtime_path_str = runtime_path.to_string_lossy(); let path_key = if cfg!(target_os = "windows") { "PATH" } else if cfg!(target_os = "macos") { "DYLD_LIBRARY_PATH" } else { "LD_LIBRARY_PATH" }; let path_var = std::env::var_os(path_key).unwrap_or_default(); let mut paths: Vec<_> = std::env::split_paths(&path_var).collect(); if !paths.contains(&runtime_path) { paths.insert(0, runtime_path.clone()); let new_path = std::env::join_paths(paths).unwrap(); std::env::set_var(path_key, new_path); println!(" [System] Added to {}: {}", path_key, runtime_path_str); } #[cfg(target_os = "windows")] { use std::os::windows::ffi::OsStrExt; let path_wide = std::ffi::OsStr::new(&runtime_path) .encode_wide() .chain(std::iter::once(0)) .collect::>(); winapi::um::winbase::SetDllDirectoryW(path_wide.as_ptr()); println!(" [System] SetDllDirectoryW set to: {}", runtime_path_str); } let (ggml_name, llama_name, libomp_name) = if cfg!(target_os = "windows") { ("ggml.dll", "llama.dll", Some("libomp140.x86_64.dll")) } else if cfg!(target_os = "macos") { ("libggml.dylib", "libllama.dylib", None) } else { ("libggml.so", "libllama.so", None) }; if let Some(omp_name) = libomp_name { let libomp_path = runtime_path.join(omp_name); let _libomp = libloading::Library::new(&libomp_path).ok(); } ensure_runtime_library_link(&runtime_path, "libgomp.so.1"); let ggml_path = runtime_path.join(ggml_name); let ggml_lib = libloading::Library::new(&ggml_path) .map_err(|e| { eprintln!( "WARNING: Failed to load {} from {:?}: {}", ggml_name, ggml_path, e ) }) .ok(); let cpu_backend_name = if cfg!(target_arch = "x86_64") { Some("libggml-cpu-x64.so") } else if cfg!(target_arch = "aarch64") { Some("libggml-cpu-arm64.so") } else { None }; let cpu_backend_lib = cpu_backend_name .and_then(|name| { let cpu_backend_path = runtime_path.join(name); match libloading::Library::new(&cpu_backend_path) { Ok(lib) => Some(lib), Err(e) => { eprintln!( "WARNING: Failed to load CPU backend {} from {:?}: {}", name, cpu_backend_path, e ); None } } }); #[cfg(target_os = "linux")] { if let Some(name) = cpu_backend_name { let backend_path = runtime_path.join(name); std::env::set_var("GGML_BACKEND_PATH", &backend_path); println!(" [System] GGML_BACKEND_PATH set to: {}", backend_path.to_string_lossy()); } } let llama_path = runtime_path.join(llama_name); let lib = libloading::Library::new(&llama_path) .unwrap_or_else(|_| panic!("Failed to load {}. Please ensure it is in the runtime/ directory.", llama_name)); unsafe extern "C" fn dummy_fn() {} let load_all_fn: GgmlBackendLoadAllFn = if let Some(ref glib) = ggml_lib { match glib.get::(b"ggml_backend_load_all") { Ok(symbol) => *symbol, Err(_) => dummy_fn, } } else { lib.get::(b"ggml_backend_load_all") .map(|s| *s) .unwrap_or(dummy_fn) }; let vulkan_path = runtime_path.join("libggml-vulkan.so"); match libloading::Library::new(&vulkan_path) { Ok(_) => println!(" [System] Verified: libggml-vulkan.so is loadable."), Err(e) => eprintln!(" [System] WARNING: libggml-vulkan.so is NOT loadable: {}. This usually means missing Vulkan drivers or runtime.", e), } let ffi = LlamaFFI { llama_backend_init: *lib.get(b"llama_backend_init").expect("llama_backend_init"), llama_backend_free: *lib.get(b"llama_backend_free").expect("llama_backend_free"), llama_model_default_params: *lib .get(b"llama_model_default_params") .expect("llama_model_default_params"), llama_model_load_from_file: *lib .get(b"llama_model_load_from_file") .expect("llama_model_load_from_file"), llama_model_free: *lib.get(b"llama_model_free").expect("llama_model_free"), llama_model_get_vocab: *lib .get(b"llama_model_get_vocab") .expect("llama_model_get_vocab"), llama_model_n_embd: *lib.get(b"llama_model_n_embd").expect("llama_model_n_embd"), llama_model_n_head: *lib.get(b"llama_model_n_head").expect("llama_model_n_head"), llama_model_n_layer: *lib .get(b"llama_model_n_layer") .expect("llama_model_n_layer"), llama_model_n_ctx: *lib.get(b"llama_n_ctx").expect("llama_n_ctx"), llama_model_n_vocab: *lib.get(b"llama_n_vocab").expect("llama_n_vocab"), llama_vocab_n_tokens: *lib .get(b"llama_vocab_n_tokens") .expect("llama_vocab_n_tokens"), llama_vocab_eos: *lib.get(b"llama_vocab_eos").expect("llama_vocab_eos"), llama_context_default_params: *lib .get(b"llama_context_default_params") .expect("llama_context_default_params"), llama_init_from_model: *lib .get(b"llama_init_from_model") .expect("llama_init_from_model"), llama_free: *lib.get(b"llama_free").expect("llama_free"), llama_batch_init: *lib.get(b"llama_batch_init").expect("llama_batch_init"), llama_batch_free: *lib.get(b"llama_batch_free").expect("llama_batch_free"), llama_decode: *lib.get(b"llama_decode").expect("llama_decode"), llama_get_embeddings: *lib .get(b"llama_get_embeddings") .expect("llama_get_embeddings"), llama_get_logits: *lib.get(b"llama_get_logits").expect("llama_get_logits"), llama_get_memory: *lib.get(b"llama_get_memory").expect("llama_get_memory"), llama_memory_clear: *lib.get(b"llama_memory_clear").expect("llama_memory_clear"), llama_memory_seq_rm: *lib .get(b"llama_memory_seq_rm") .expect("llama_memory_seq_rm"), llama_memory_seq_pos_max: *lib .get(b"llama_memory_seq_pos_max") .expect("llama_memory_seq_pos_max"), llama_sampler_init_temp: *lib .get(b"llama_sampler_init_temp") .expect("llama_sampler_init_temp"), llama_sampler_sample: *lib .get(b"llama_sampler_sample") .expect("llama_sampler_sample"), llama_sampler_free: *lib.get(b"llama_sampler_free").expect("llama_sampler_free"), ggml_backend_load_all: load_all_fn, }; let original_cwd = std::env::current_dir().ok(); if original_cwd.is_some() { let _ = std::env::set_current_dir(&runtime_path); } (ffi.ggml_backend_load_all)(); (ffi.llama_backend_init)(); if let Some(cwd) = original_cwd { let _ = std::env::set_current_dir(cwd); } std::mem::forget(lib); if let Some(glib) = ggml_lib { std::mem::forget(glib); } if let Some(cpu_backend_lib) = cpu_backend_lib { std::mem::forget(cpu_backend_lib); } ffi } }) } pub fn cleanup() { if let Some(ffi) = FFI.get() { unsafe { (ffi.llama_backend_free)(); } } } pub struct LlamaModel { pub ptr: LlamaModelPtr, pub vocab: LlamaVocabPtr, pub n_embd: usize, pub n_head: usize, pub n_layer: usize, pub n_ctx: usize, pub n_vocab: usize, pub eos_token: LlamaToken, } unsafe impl Send for LlamaModel {} unsafe impl Sync for LlamaModel {} impl LlamaModel { pub fn load(path: &Path, n_gpu_layers: i32) -> Result { let ffi = get_ffi(); unsafe { let mut params = (ffi.llama_model_default_params)(); params.n_gpu_layers = n_gpu_layers; let c_path = std::ffi::CString::new(path.to_str().unwrap()).map_err(|e| e.to_string())?; let ptr = (ffi.llama_model_load_from_file)(c_path.as_ptr(), params); if ptr.is_null() { return Err("Failed to load model".to_string()); } let vocab = (ffi.llama_model_get_vocab)(ptr); let n_vocab = (ffi.llama_vocab_n_tokens)(vocab) as usize; let n_embd = (ffi.llama_model_n_embd)(ptr) as usize; let n_head = (ffi.llama_model_n_head)(ptr) as usize; let n_layer = (ffi.llama_model_n_layer)(ptr) as usize; let eos_token = (ffi.llama_vocab_eos)(vocab); Ok(Self { ptr, vocab, n_embd, n_head, n_layer, n_ctx: 32768, n_vocab, eos_token, }) } } } impl Drop for LlamaModel { fn drop(&mut self) { unsafe { if let Some(ffi) = FFI.get() { if !self.ptr.is_null() { (ffi.llama_model_free)(self.ptr); } } } } } impl Clone for LlamaModel { fn clone(&self) -> Self { Self { ptr: self.ptr, vocab: self.vocab, n_embd: self.n_embd, n_head: self.n_head, n_layer: self.n_layer, n_ctx: self.n_ctx, n_vocab: self.n_vocab, eos_token: self.eos_token, } } } pub struct LlamaContext { pub ptr: LlamaContextPtr, pub model: std::mem::ManuallyDrop, } unsafe impl Send for LlamaContext {} unsafe impl Sync for LlamaContext {} impl LlamaContext { pub fn new( model: &LlamaModel, n_ctx: u32, n_batch: u32, embeddings: i32, n_threads: i32, ) -> Result { let ffi = get_ffi(); unsafe { let mut params = (ffi.llama_context_default_params)(); params.n_ctx = n_ctx; params.n_batch = n_batch; params.n_ubatch = 512; params.n_seq_max = 1; params.embeddings = embeddings != 0; params.flash_attn_type = 1; params.offload_kqv = true; params.no_perf = true; if n_threads > 0 { params.n_threads = n_threads; } else { let cpu_count = std::thread::available_parallelism() .map(|p| p.get() as c_int) .unwrap_or(4); params.n_threads = (cpu_count / 2).min(4); } let ptr = (ffi.llama_init_from_model)(model.ptr, params); if ptr.is_null() { return Err("Failed to create context".to_string()); } Ok(Self { ptr, model: std::mem::ManuallyDrop::new(model.clone()), }) } } pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), String> { let ffi = get_ffi(); unsafe { let result = (ffi.llama_decode)(self.ptr, batch.batch); if result != 0 { return Err(format!("llama_decode failed: {}", result)); } Ok(()) } } pub fn get_embeddings(&self) -> &[f32] { let ffi = get_ffi(); unsafe { std::slice::from_raw_parts((ffi.llama_get_embeddings)(self.ptr), self.model.n_embd) } } pub fn get_embedding_at(&self, batch_index: usize) -> &[f32] { let ffi = get_ffi(); unsafe { let base_ptr = (ffi.llama_get_embeddings)(self.ptr); let offset = batch_index * self.model.n_embd; std::slice::from_raw_parts(base_ptr.add(offset), self.model.n_embd) } } pub fn get_logits(&self) -> &[f32] { let ffi = get_ffi(); unsafe { std::slice::from_raw_parts((ffi.llama_get_logits)(self.ptr), self.model.n_vocab) } } pub fn get_logits_ith(&self, i: usize) -> &[f32] { let ffi = get_ffi(); unsafe { let base_ptr = (ffi.llama_get_logits)(self.ptr); std::slice::from_raw_parts(base_ptr.add(i * self.model.n_vocab), self.model.n_vocab) } } pub fn clear_kv_cache(&self) -> bool { let ffi = get_ffi(); unsafe { let mem = (ffi.llama_get_memory)(self.ptr); let result = (ffi.llama_memory_seq_rm)(mem, -1, 0, -1); if !result { eprintln!("WARNING: clear_kv_cache failed for ctx {:?}", self.ptr); } result } } pub fn debug_seq_pos_max(&self, seq_id: c_int) -> c_int { let ffi = get_ffi(); unsafe { let mem = (ffi.llama_get_memory)(self.ptr); let result = (ffi.llama_memory_seq_pos_max)(mem, seq_id); eprintln!( "DEBUG: [debug_seq_pos_max] mem={:?}, seq_id={}, result={}", mem, seq_id, result ); result } } } impl Drop for LlamaContext { fn drop(&mut self) { unsafe { if let Some(ffi) = FFI.get() { if !self.ptr.is_null() { (ffi.llama_free)(self.ptr); } } } } } pub struct LlamaBatch { batch: llama_batch, n_tokens_max: usize, n_embd: usize, _n_pos_per_embd: usize, _embd_buffer: Vec, _pos_buffer: Vec, _seq_id_buffers: Vec>, _n_seq_id_buffer: Vec, _logits_buffer: Vec, } impl LlamaBatch { pub fn new( n_tokens_max: usize, n_embd: usize, n_seq_max: usize, n_pos_per_embd: usize, ) -> Self { let ffi = get_ffi(); unsafe { let batch = (ffi.llama_batch_init)(n_tokens_max as c_int, n_embd as c_int, n_seq_max as c_int); Self { batch, n_tokens_max, n_embd, _n_pos_per_embd: n_pos_per_embd, _embd_buffer: Vec::new(), _pos_buffer: Vec::new(), _seq_id_buffers: Vec::new(), _n_seq_id_buffer: Vec::new(), _logits_buffer: Vec::new(), } } } pub fn set_embd(&mut self, prompt_embeds: &[f32], pos_arr: &[i32], seq_id: i32) { let n_tokens = prompt_embeds.len() / self.n_embd; unsafe { std::ptr::copy_nonoverlapping( prompt_embeds.as_ptr(), self.batch.embd, prompt_embeds.len(), ); let max_pos = self.n_tokens_max; if pos_arr.len() > max_pos { eprintln!( "WARNING: pos_arr length {} exceeds batch capacity {}. Truncating!", pos_arr.len(), max_pos ); } std::ptr::copy_nonoverlapping( pos_arr.as_ptr(), self.batch.pos, pos_arr.len().min(max_pos), ); } for i in 0..n_tokens { unsafe { *self.batch.n_seq_id.add(i) = 1; let seq_ids = *self.batch.seq_id.add(i); *seq_ids.add(0) = seq_id; *self.batch.logits.add(i) = if i == n_tokens - 1 { 1 } else { 0 }; } } self.batch.n_tokens = n_tokens as c_int; } pub fn clear(&mut self) { self.batch.n_tokens = 0; } pub fn n_tokens(&self) -> usize { self.batch.n_tokens as usize } pub fn batch_ptr(&mut self) -> *mut llama_batch { &mut self.batch } } pub struct LlamaSampler { _ptr: LlamaSamplerPtr, n_vocab: usize, _neg_inf: f32, temperature: f32, top_k: usize, top_p: f32, min_p: f32, repeat_penalty: f32, frequency_penalty: f32, presence_penalty: f32, penalty_last_n: usize, token_history: std::cell::RefCell>, rng: std::cell::RefCell, } impl LlamaSampler { pub fn new(n_vocab: usize, temperature: f32, top_k: i32, top_p: f32, seed: u64) -> Self { use rand::SeedableRng; Self { _ptr: std::ptr::null_mut(), n_vocab, _neg_inf: -1e9, temperature, top_k: top_k as usize, top_p, min_p: 0.0, repeat_penalty: 1.0, frequency_penalty: 0.0, presence_penalty: 0.0, penalty_last_n: 64, token_history: std::cell::RefCell::new(Vec::new()), rng: std::cell::RefCell::new(rand::rngs::StdRng::seed_from_u64(seed)), } } pub fn greedy(n_vocab: usize) -> Self { use rand::SeedableRng; Self { _ptr: std::ptr::null_mut(), n_vocab, _neg_inf: -1e9, temperature: 0.0, top_k: 0, top_p: 1.0, min_p: 0.0, repeat_penalty: 1.0, frequency_penalty: 0.0, presence_penalty: 0.0, penalty_last_n: 64, token_history: std::cell::RefCell::new(Vec::new()), rng: std::cell::RefCell::new(rand::rngs::StdRng::seed_from_u64(42)), } } pub fn with_penalties( mut self, min_p: f32, repeat_penalty: f32, frequency_penalty: f32, presence_penalty: f32, penalty_last_n: usize, ) -> Self { self.min_p = min_p; self.repeat_penalty = repeat_penalty; self.frequency_penalty = frequency_penalty; self.presence_penalty = presence_penalty; self.penalty_last_n = penalty_last_n; self } pub fn accept(&self, token: LlamaToken) { let mut history = self.token_history.borrow_mut(); history.push(token); if history.len() > self.penalty_last_n { let excess = history.len() - self.penalty_last_n; history.drain(0..excess); } } fn apply_penalties(&self, logits: &mut [f32]) { if self.repeat_penalty == 1.0 && self.frequency_penalty == 0.0 && self.presence_penalty == 0.0 { return; } let history = self.token_history.borrow(); let mut token_counts: std::collections::HashMap = std::collections::HashMap::new(); for &token in history.iter() { *token_counts.entry(token).or_insert(0) += 1; } // 惩罚豁免名单:EOS(2150)、PAD(2148)、BOS(2149)、TTS_EOS(151673) let penalty_exempt: std::collections::HashSet = [2150, 2148, 2149, 151673].iter().cloned().collect(); for (&token, &count) in token_counts.iter() { // 跳过豁免名单中的 token if penalty_exempt.contains(&token) { continue; } let idx = token as usize; if idx < logits.len() { if self.repeat_penalty != 1.0 { if logits[idx] > 0.0 { logits[idx] /= self.repeat_penalty; } else { logits[idx] *= self.repeat_penalty; } } logits[idx] -= self.frequency_penalty * count as f32; logits[idx] -= self.presence_penalty; } } } fn apply_min_p(&self, probs: &mut Vec<(usize, f32)>) { if self.min_p <= 0.0 || probs.is_empty() { return; } let max_prob = probs.iter().map(|(_, p)| *p).fold(0.0, f32::max); let threshold = max_prob * self.min_p; probs.retain(|(_, p)| *p >= threshold); } pub fn sample( &self, ctx: &LlamaContext, idx: i32, limit_start: Option, limit_end: Option, ) -> LlamaToken { self.sample_with_allow(ctx, idx, limit_start, limit_end, None) } pub fn sample_with_allow( &self, ctx: &LlamaContext, idx: i32, limit_start: Option, limit_end: Option, allow_tokens: Option<&[LlamaToken]>, ) -> LlamaToken { let ffi = get_ffi(); unsafe { let offset = if idx >= 0 { idx as usize } else { 0 }; let base_ptr = (ffi.llama_get_logits)(ctx.ptr); let logits_ptr = base_ptr.add(offset * self.n_vocab); let logits_raw = std::slice::from_raw_parts(logits_ptr, self.n_vocab); let mut logits: Vec = logits_raw.to_vec(); let start = limit_start.unwrap_or(0); let end = limit_end.unwrap_or(self.n_vocab).min(self.n_vocab); let mut mask = vec![true; self.n_vocab]; for m in mask.iter_mut().take(end).skip(start) { *m = false; } if let Some(allow) = allow_tokens { for &token in allow { let t = token as usize; if t < self.n_vocab { mask[t] = false; } } } for (i, &m) in mask.iter().enumerate() { if m { logits[i] = self._neg_inf; } } self.apply_penalties(&mut logits); if self.temperature <= 0.0 { let mut max_val = f32::NEG_INFINITY; let mut max_idx = 0; for (i, &val) in logits.iter().enumerate() { if val > max_val { max_val = val; max_idx = i; } } self.accept(max_idx as LlamaToken); return max_idx as LlamaToken; } let mut candidates: Vec<(usize, f32)> = logits .iter() .enumerate() .filter(|(_, &v)| v > f32::NEG_INFINITY / 2.0) .map(|(i, &v)| (i, v)) .collect(); candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); if self.top_k > 0 && self.top_k < candidates.len() { candidates.truncate(self.top_k); } let max_logit = candidates.first().map(|(_, l)| *l).unwrap_or(0.0); let mut probs: Vec<(usize, f32)> = candidates .iter() .map(|(idx, logit)| { let scaled = (*logit - max_logit) / self.temperature; (*idx, scaled.exp()) }) .collect(); let sum: f32 = probs.iter().map(|(_, p)| *p).sum(); if sum > 0.0 { for (_, p) in probs.iter_mut() { *p /= sum; } } self.apply_min_p(&mut probs); if self.top_p < 1.0 { let mut cumsum = 0.0; let mut cutoff_idx = probs.len(); for (i, (_, p)) in probs.iter().enumerate() { cumsum += p; if cumsum >= self.top_p { cutoff_idx = i + 1; break; } } probs.truncate(cutoff_idx); let new_sum: f32 = probs.iter().map(|(_, p)| *p).sum(); if new_sum > 0.0 { for (_, p) in probs.iter_mut() { *p /= new_sum; } } } use rand::Rng; let r: f32 = self.rng.borrow_mut().gen(); let mut cumsum = 0.0; for (idx, p) in probs.iter() { cumsum += p; if r < cumsum { self.accept(*idx as LlamaToken); return *idx as LlamaToken; } } let result = probs .first() .map(|(idx, _)| *idx as LlamaToken) .unwrap_or(start as LlamaToken); self.accept(result); result } } /// 采样时对静音 token 施加惩罚 /// silent_penalty: 静音惩罚系数(0-1),降低静音 token 的概率 /// silent_threshold: 静音 token 的阈值(code_0 < silent_threshold 被视为静音) pub fn sample_with_silent_penalty( &self, ctx: &LlamaContext, idx: i32, limit_start: Option, limit_end: Option, allow_tokens: Option<&[LlamaToken]>, silent_penalty: f32, silent_threshold: usize, ) -> LlamaToken { let ffi = get_ffi(); unsafe { let offset = if idx >= 0 { idx as usize } else { 0 }; let base_ptr = (ffi.llama_get_logits)(ctx.ptr); let logits_ptr = base_ptr.add(offset * self.n_vocab); let logits_raw = std::slice::from_raw_parts(logits_ptr, self.n_vocab); let mut logits: Vec = logits_raw.to_vec(); let start = limit_start.unwrap_or(0); let end = limit_end.unwrap_or(self.n_vocab).min(self.n_vocab); // 创建掩码 let mut mask = vec![true; self.n_vocab]; for m in mask.iter_mut().take(end).skip(start) { *m = false; } if let Some(allow) = allow_tokens { for &token in allow { let t = token as usize; if t < self.n_vocab { mask[t] = false; } } } for (i, &m) in mask.iter().enumerate() { if m { logits[i] = self._neg_inf; } } // 对静音 token 施加惩罚(降低 logits) if silent_penalty > 0.0 { for i in 0..silent_threshold.min(end) { if !mask[i] { logits[i] -= silent_penalty; } } } self.apply_penalties(&mut logits); if self.temperature <= 0.0 { let mut max_val = f32::NEG_INFINITY; let mut max_idx = 0; for (i, &val) in logits.iter().enumerate() { if val > max_val { max_val = val; max_idx = i; } } self.accept(max_idx as LlamaToken); return max_idx as LlamaToken; } let mut candidates: Vec<(usize, f32)> = logits .iter() .enumerate() .filter(|(_, &v)| v > f32::NEG_INFINITY / 2.0) .map(|(i, &v)| (i, v)) .collect(); candidates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); if self.top_k > 0 && self.top_k < candidates.len() { candidates.truncate(self.top_k); } let max_logit = candidates.first().map(|(_, l)| *l).unwrap_or(0.0); let mut probs: Vec<(usize, f32)> = candidates .iter() .map(|(idx, logit)| { let scaled = (*logit - max_logit) / self.temperature; (*idx, scaled.exp()) }) .collect(); let sum: f32 = probs.iter().map(|(_, p)| *p).sum(); if sum > 0.0 { for (_, p) in probs.iter_mut() { *p /= sum; } } self.apply_min_p(&mut probs); if self.top_p < 1.0 { let mut cumsum = 0.0; let mut cutoff_idx = probs.len(); for (i, (_, p)) in probs.iter().enumerate() { cumsum += p; if cumsum > self.top_p { cutoff_idx = i + 1; break; } } probs.truncate(cutoff_idx); } use rand::Rng; let r: f32 = self.rng.borrow_mut().gen(); let mut cumsum = 0.0; for (idx, p) in probs.iter() { cumsum += p; if r < cumsum { self.accept(*idx as LlamaToken); return *idx as LlamaToken; } } let result = probs .first() .map(|(idx, _)| *idx as LlamaToken) .unwrap_or(start as LlamaToken); self.accept(result); result } } } impl Drop for LlamaSampler { fn drop(&mut self) {} }