修复了编译错误和onnx运行时链接错误

This commit is contained in:
:NoBody: committed 2026-05-29 16:58:55 +08:00
commit 7d8b3370fa
48 files changed
+26105

No files matched your search

+159
View File
@@ -0,0 +1,159 @@
name: Build and Release
on:
push:
tags:
- 'v*'
workflow_dispatch:
permissions:
contents: write
jobs:
build:
name: Build - ${{ matrix.platform.os_name }} - ${{ matrix.platform.backend }}
runs-on: ${{ matrix.platform.os }}
strategy:
matrix:
platform:
# Windows Vulkan
- os_name: Windows-x64
os: windows-latest
target: x86_64-pc-windows-msvc
backend: vulkan
features: vulkan
bin: qwen3_tts.exe
name: qwen3-tts-windows-x64-vulkan.zip
llama_url: https://github.com/ggerganov/llama.cpp/releases/download/b7885/llama-b7885-bin-win-vulkan-x64.zip
ort_url: https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-win-x64-1.23.2.zip
# Windows CUDA 12
- os_name: Windows-x64
os: windows-latest
target: x86_64-pc-windows-msvc
backend: cuda
features: cuda
bin: qwen3_tts.exe
name: qwen3-tts-windows-x64-cuda12.zip
llama_url: https://github.com/ggerganov/llama.cpp/releases/download/b7885/llama-b7885-bin-win-cuda-12.4-x64.zip
ort_url: https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-win-x64-gpu-1.23.2.zip
# Linux Vulkan
- os_name: Linux-x64
os: ubuntu-latest
target: x86_64-unknown-linux-gnu
backend: vulkan
features: vulkan
bin: qwen3_tts
name: qwen3-tts-linux-x64-vulkan.tar.gz
llama_url: https://github.com/ggerganov/llama.cpp/releases/download/b7885/llama-b7885-bin-ubuntu-vulkan-x64.tar.gz
ort_url: https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-linux-x64-1.23.2.tgz
# macOS Metal
- os_name: macOS-ARM64
os: macos-14
target: aarch64-apple-darwin
backend: metal
features: "" # No specific feature needed for Metal? Or 'metal'? Cargo.toml doesn't have 'metal'.
bin: qwen3_tts
name: qwen3-tts-macos-arm64.tar.gz
llama_url: https://github.com/ggerganov/llama.cpp/releases/download/b7885/llama-b7885-bin-macos-arm64.tar.gz
ort_url: https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-osx-arm64-1.23.2.tgz
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.platform.target }}
- name: Install Linux Dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libasound2-dev pkg-config libssl-dev
- name: Build Binary
run: cargo build --release --target ${{ matrix.platform.target }} --features "${{ matrix.platform.features }}"
- name: Prepare Distribution
shell: bash
run: |
mkdir -p dist
mkdir -p dist/runtime
mkdir -p dist/models
mkdir -p dist/speakers
# Copy Binary
cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }} dist/
# Copy Readme
cp README.md dist/
# Copy Speakers
cp -r speakers/* dist/speakers/
- name: Download and Bundle Runtimes (Windows)
if: runner.os == 'Windows'
shell: powershell
run: |
# Download Llama.cpp
$llamaUrl = "${{ matrix.platform.llama_url }}"
Invoke-WebRequest -Uri $llamaUrl -OutFile "llama.zip"
Expand-Archive -Path "llama.zip" -DestinationPath "dist/runtime" -Force
rm llama.zip
# Download ONNX Runtime
$ortUrl = "${{ matrix.platform.ort_url }}"
Invoke-WebRequest -Uri $ortUrl -OutFile "ort.zip"
Expand-Archive -Path "ort.zip" -DestinationPath "ort_temp" -Force
Copy-Item "ort_temp/*/lib/*.dll" -Destination "dist/runtime" -Recurse
rm ort.zip
rm ort_temp -Recurse -Force
- name: Download and Bundle Runtimes (Unix)
if: runner.os != 'Windows'
shell: bash
run: |
# Download Llama.cpp
# Check if URL exists (optional but good practice)
wget -O llama.tar.gz "${{ matrix.platform.llama_url }}"
tar -xzvf llama.tar.gz -C dist/runtime
rm llama.tar.gz
# Download ONNX Runtime
wget -O ort.tgz "${{ matrix.platform.ort_url }}"
mkdir ort_temp
tar -xzvf ort.tgz -C ort_temp
# Copy libs
find ort_temp -name "*.so*" -exec cp {} dist/runtime/ \;
find ort_temp -name "*.dylib" -exec cp {} dist/runtime/ \;
rm ort.tgz
rm -rf ort_temp
- name: Archive Release (Windows)
if: runner.os == 'Windows'
shell: powershell
run: |
Compress-Archive -Path dist/* -DestinationPath ${{ matrix.platform.name }}
- name: Archive Release (Unix)
if: runner.os != 'Windows'
shell: bash
run: |
tar -czvf ${{ matrix.platform.name }} -C dist .
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.platform.name }}
path: ${{ matrix.platform.name }}
- name: Create Release
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
files: ${{ matrix.platform.name }}
+36
View File
@@ -0,0 +1,36 @@
# Rust
/target
/debug
/release
Cargo.lock
# Models and Large Assets
/models
*.onnx
*.gguf
*.pth
*.pt
*.bin
*.wav
*.qvoice
# Runtime binaries
/runtime
*.dll
*.exe
*.so
*.dylib
# IDE
.idea
.vscode
*.iml
.trae
# OS
Thumbs.db
Desktop.ini
.DS_Store
#test
.flake.nix.swp
/down
+58
View File
@@ -0,0 +1,58 @@
[package]
name = "qwen3-tts"
version = "0.1.7"
edition = "2021"
description = "Rust implementation of Qwen3-TTS with ONNX Runtime"
license = "MIT OR Apache-2.0"
repository = "https://github.com/cgisky1980/Qwen3-TTS-Rust"
readme = "README.md"
[lib]
path = "src/lib.rs"
[dependencies]
tokio = { version = "1", features = ["full"] }
ndarray = "0.17"
ndarray-npy = "0.8"
rand = "0.8"
ort = { version = "2.0.0-rc.11", features = ["ndarray", "load-dynamic", "cuda", "directml","rocm"] }
tokenizers = "0.22"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
hound = "3.5"
cpal = "0.15"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
libc = "0.2"
libloading = "0.8"
clap = { version = "4.5", features = ["derive"] }
mel_spec = "0.3"
rustfft = "6.4.1"
reqwest = { version = "0.11", features = ["blocking", "stream"] }
indicatif = "0.17"
futures-util = "0.3"
zip = "2.2"
tar = "0.4"
flate2 = "1.0"
num_cpus = "1.16"
axum = { version = "0.7", features = ["ws", "multipart"] }
axum-server = "0.7"
tower = "0.5"
tower-http = { version = "0.6", features = ["fs", "cors"] }
tokio-tungstenite = "0.26"
base64 = "0.22"
[features]
cuda = []
vulkan = []
profile = [] # 启用性能分析计时
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3", features = ["consoleapi", "fileapi", "handleapi", "winbase"] }
[profile.release]
opt-level = 3
lto = true
codegen-units = 1
+187
View File
@@ -0,0 +1,187 @@
# Qwen3-TTS Rust
[中文](README.md) | [English](docs/README_EN.md) | [日本語](docs/README_JA.md) | [Korean](docs/README_KO.md) | [Français](docs/README_FR.md) | [Español](docs/README_ES.md) | [Italiano](docs/README_IT.md) | [Deutsch](docs/README_DE.md) | [Русский](docs/README_RU.md) | [Português](docs/README_PT.md)
本项目是 Qwen3-TTS 的极致性能实现,核心突破在于 **“指令驱动 (Instruction-Driven)”** 与 **“零样本自定义音色 (Custom Speakers)”** 的深度集成。通过 Rust 的内存安全特性与 llama.cpp/ONNX 的高效推理,为您提供工业级的文本转语音解决方案。
## 🚀 核心特性
### 1. 极致性能与流式响应
- **并发流式解码**:采用 4 帧 (64 codes) 粒度的并发解码策略,首字延迟低至 300ms,实现“边想边说”的流畅体验。
- **硬件加速**:默认启用 **Vulkan** (Windows/Linux) 和 **Metal** (macOS) 加速,显著提升推理速度。
- **自动运行时管理**:零配置环境,自动下载并配置 `llama.cpp` (b7885) 和 `onnxruntime`,开箱即用。
### 2. 灵活的说话人管理
- **自动扫描与缓存**:启动时自动加载 `speakers/` 目录下的音色文件。
- **多种选择方式**:支持通过 CLI 参数 `--speaker <name>` 或 `--voice-file <path>` 灵活选择说话人。
- **智能回退**:若指定说话人不存在,自动回退至默认音色 (vivian),确保系统稳定性。
### 3. 精准的指令控制
- **指令驱动**:支持在文本中嵌入 `[高兴]`、`[悲伤]` 等情感指令,实时调整演绎风格。
- **EOS 对齐**:完美对齐 Qwen3 的停止逻辑,支持多种 EOS token 检测,杜绝生成末尾的静音或乱码。
## 📊 性能基准 (Benchmarks)
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **测试环境**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. 显存占用约 0.7-1.5GB.
- **数据来源**: Windows 平台 10 轮生成平均值。
- **最佳性能**: RTF 0.553 (CUDA + Q5KM),即生成 1 秒音频仅需 0.553 秒。
## 🛠️ 快速上手
### 1. 基础生成
使用默认说话人生成语音:
```powershell
cargo run --bin qwen3_tts -- --text "你好,欢迎使用 Qwen3-TTS Rust!"
```
### 2. 指定说话人
使用预设或自定义说话人:
```powershell
# 使用名称 (需在 speakers/ 目录下存在对应的 .json 文件)
cargo run --bin qwen3_tts -- --text "今天天气不错。" --speaker dylan
# 使用指定文件路径
cargo run --bin qwen3_tts -- --text "我是自定义音色。" --voice-file "path/to/my_voice.json"
```
### 3. 克隆新音色
只需 3-10 秒的参考音频即可克隆音色:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "参考音频对应的文本内容" `
--save-voice "speakers/my_voice.json" `
--text "新音色已保存,现在可以直接使用了!"
```
### 4. 风格/情感控制 (Instruction)
通过 `--instruction` 参数实时改变说话语气:
```powershell
# 悲伤语气
cargo run --bin qwen3_tts -- --text "对不起,我不是故意的..." --instruction "悲伤啜泣,非常难过"
# 开心语气
cargo run --bin qwen3_tts -- --text "太棒了!我们成功了!" --instruction "开心激动,语速稍快"
```
### 5. 高级配置
```powershell
cargo run --bin qwen3_tts -- `
--text "长文本生成测试。" `
--max-steps 1024 ` # 调整最大生成长度
--output "output.wav" # 指定输出文件名
```
### 6. 采样参数调整
通过采样参数控制生成的随机性和多样性:
```powershell
# 调整温度 (更高 = 更随机,0.0 = 贪婪)
cargo run --bin qwen3_tts -- --text "测试文本" --temperature 0.8
# 使用固定种子复现结果
cargo run --bin qwen3_tts -- --text "测试文本" --seed 12345
# 完整参数示例
cargo run --bin qwen3_tts -- `
--text "测试采样参数。" `
--temperature 0.7 ` # 温度 (默认 0.7)
--top-k 40 ` # Top-K 采样 (默认 40)
--top-p 0.9 ` # Top-P 核采样 (默认 0.9)
--seed 42 # 随机种子 (可选)
```
## 📦 作为 Rust 库使用
在 `Cargo.toml` 中添加依赖:
```toml
[dependencies]
# 本地引用
qwen3-tts = { path = "." }
# 或从 Git 引用
# qwen3-tts = { git = "https://github.com/cgisky1980/Qwen3-TTS-Rust" }
```
### 基础用法
```rust
use qwen3_tts::{TtsEngine, VoiceFile, SamplerConfig};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 1. 初始化引擎
let mut engine = TtsEngine::new("models", "none").await?;
// 2. 配置采样参数 (可选)
let sampler_config = SamplerConfig::new(0.7, 40, 0.9, None);
engine.set_sampler_config(sampler_config);
// 3. 加载音色
let voice = VoiceFile::load("speakers/sohee.json")?;
// 4. 生成语音
let audio = engine.generate_with_voice(
"你好,欢迎使用 Qwen3-TTS!",
&voice,
None, // instruction
)?;
// 5. 保存音频
audio.save_wav("output.wav")?;
Ok(())
}
```
### 从参考音频创建音色
```rust
use qwen3_tts::TtsEngine;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut engine = TtsEngine::new("models", "none").await?;
// 从参考音频提取音色
let voice = engine.create_voice_file(
"reference.wav",
"参考音频对应的文本".to_string(),
)?;
// 保存音色文件
voice.save("speakers/my_voice.json")?;
// 使用新音色生成
let audio = engine.generate_with_voice("测试新音色", &voice, None)?;
audio.save_wav("output.wav")?;
Ok(())
}
```
## �� 目录结构
系统首次运行会自动构建如下结构:
```text
.
├── models/ # 模型文件 (GGUF, ONNX, Tokenizer)
├── runtime/ # 自动下载的依赖库 (dll, so, dylib)
└── speakers/ # 用户自定义音色
```
## 📜 许可证与致谢
- 基于 **MIT / Apache 2.0** 许可证。
- 感谢 [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) 官方仓库提供的模型与技术基座。
- 感谢 [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) 提供的推理流程启发。
+90
View File
@@ -0,0 +1,90 @@
<<<<<<< nkmumlpy 2a5b03c6 "chore: 版本号更新到 0.1.7" (rebase destination)
fn main() {}
||||||| sxlwkwwz 32ed8f03 "fix: improve library extraction and bake RPATH into binary (#8)" (parents of rebased revision)
use std::env;
use std::path::PathBuf;
fn main() {
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
if target_os == "linux" {
// Set the RPATH for the binary
println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/runtime");
// Optional: Automatically link the runtime folder into the target directory
// so 'cargo run' works without extra RPATHs
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let profile_dir = out_dir.join("../../../"); // Points to target/debug or target/release
let project_runtime =
PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("runtime");
let target_runtime = profile_dir.join("runtime");
if project_runtime.exists() && !target_runtime.exists() {
#[cfg(unix)]
let _ = std::os::unix::fs::symlink(project_runtime, target_runtime);
}
}
}
=======
use std::env;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
#[cfg(unix)]
fn ensure_symlink(target: &Path, link: &Path) {
if link.exists() {
return;
}
if let Some(parent) = link.parent() {
let _ = fs::create_dir_all(parent);
}
let _ = std::os::unix::fs::symlink(target, link);
}
#[cfg(not(unix))]
fn ensure_symlink(_target: &Path, _link: &Path) {}
fn main() {
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
if target_os == "linux" {
// Set the RPATH for the binary
//println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/runtime");
// Optional: Automatically link the runtime folder into the target directory
// so 'cargo run' works without extra RPATHs
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let profile_dir = out_dir.join("../../../"); // Points to target/debug or target/release
let project_runtime =
PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("runtime");
let target_runtime = profile_dir.join("runtime");
if project_runtime.exists() && !target_runtime.exists() {
#[cfg(unix)]
let _ = std::os::unix::fs::symlink(&project_runtime, target_runtime);
}
if project_runtime.exists() {
for lib_name in ["libstdc++.so.6", "libgomp.so.1"] {
if let Ok(output) = Command::new("gcc")
.arg(format!("-print-file-name={}", lib_name))
.output()
{
if output.status.success() {
let lib_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
let lib_path = PathBuf::from(lib_path);
if lib_path.exists() && lib_path.file_name().is_some() {
let link_path = project_runtime.join(lib_path.file_name().unwrap());
ensure_symlink(&lib_path, &link_path);
}
}
}
}
}
}
}
>>>>>>> xruvuyky 5f2a399a "修复了编译错误和onnx运行时链接错误" (rebased revision)
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+187
View File
@@ -0,0 +1,187 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📦 Using as a Rust Library
Add dependency in your `Cargo.toml`:
```toml
[dependencies]
# Local reference
qwen3-tts = { path = "." }
# Or from Git
# qwen3-tts = { git = "https://github.com/cgisky1980/Qwen3-TTS-Rust" }
```
### Basic Usage
```rust
use qwen3_tts::{TtsEngine, VoiceFile, SamplerConfig};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 1. Initialize engine
let mut engine = TtsEngine::new("models", "none").await?;
// 2. Configure sampling (optional)
let sampler_config = SamplerConfig::new(0.7, 40, 0.9, None);
engine.set_sampler_config(sampler_config);
// 3. Load voice
let voice = VoiceFile::load("speakers/sohee.json")?;
// 4. Generate speech
let audio = engine.generate_with_voice(
"Hello, welcome to Qwen3-TTS!",
&voice,
None, // instruction
)?;
// 5. Save audio
audio.save_wav("output.wav")?;
Ok(())
}
```
### Create Voice from Reference Audio
```rust
use qwen3_tts::TtsEngine;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut engine = TtsEngine::new("models", "none").await?;
// Extract voice from reference audio
let voice = engine.create_voice_file(
"reference.wav",
"Text corresponding to reference audio".to_string(),
)?;
// Save voice file
voice.save("speakers/my_voice.json")?;
// Generate with new voice
let audio = engine.generate_with_voice("Testing new voice", &voice, None)?;
audio.save_wav("output.wav")?;
Ok(())
}
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
+117
View File
@@ -0,0 +1,117 @@
# Qwen3-TTS Rust
[简体中文](../README.md) | [English](README_EN.md) | [日本語](README_JA.md) | [Korean](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Italiano](README_IT.md) | [Deutsch](README_DE.md) | [Русский](README_RU.md) | [Português](README_PT.md)
This project is the ultimate performance implementation of Qwen3-TTS. The core breakthroughs are the deep integration of **"Instruction-Driven"** synthesis and **"Zero-shot Custom Speakers (Cloning)"**. Leveraging Rust's memory safety and the efficient inference of llama.cpp/ONNX, it provides an industrial-grade text-to-speech solution.
## 🚀 Core Features
### 1. Extreme Performance & Streaming
- **Concurrent Streaming Decoding**: Uses a 4-frame (64 codes) granularity for concurrent decoding, determining first-token latency as low as 300ms for a "speech-while-thinking" experience.
- **Hardware Acceleration**: **Vulkan** (Windows/Linux) and **Metal** (macOS) acceleration are enabled by default, significantly boosting inference speed.
- **Automatic Runtime Management**: Zero-config environment; automatically downloads and configures `llama.cpp` (b7885) and `onnxruntime`, ready to use out of the box.
### 2. Flexible Speaker Management
- **Auto-Scan & Cache**: Automatically loads voice files from the `speakers/` directory on startup.
- **Versatile Selection**: Supports flexible speaker selection via CLI arguments `--speaker <name>` or `--voice-file <path>`.
- **Smart Fallback**: Automatically falls back to the default voice (vivian) if the specified speaker is not found, ensuring system stability.
### 3. Precise Instruction Control
- **Instruction-Driven**: Supports embedding emotion tags like `[Happy]`, `[Sad]` in the text to adjust the delivery style in real-time.
- **EOS Alignment**: Perfectly aligned with Qwen3's stop logic, supporting multiple EOS token detections to prevent generation of trailing silence or artifacts.
## 📊 Benchmarks
| Backend | Model (GGUF) | RTF (Real-Time Factor) | Avg Time (ms) | Avg Audio (s) | Status |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **CUDA** | Q5_K_M | **0.553** | 1162.6 | 2.19 | OK |
| **Vulkan** | Q5_K_M | 0.598 | 1285.4 | 2.19 | OK |
| **CUDA** | Q8_0 | 0.640 | 1523.4 | 2.44 | OK |
| **Vulkan** | Q8_0 | 0.638 | 1502.0 | 2.44 | OK |
| **CPU** | Q5_K_M | 1.677 | 2823.4 | 1.96 | OK |
| **CPU** | Q8_0 | 1.866 | 4160.1 | 2.51 | OK |
- **Test Environment**: Intel Core i9-13980HX, NVIDIA RTX 2080 Ti. VRAM Usage ~0.7-1.5GB.
- **Data Source**: Average of 10 generations on Windows.
- **Best Performance**: RTF 0.553 (CUDA + Q5KM), meaning 1 second of audio takes only 0.553 seconds to generate.
## 🛠️ Quick Start
### 1. Basic Generation
Generate speech using the default speaker:
```powershell
cargo run --bin qwen3_tts -- --text "Hello, welcome to use Qwen3-TTS Rust!"
```
### 2. Specify Speaker
Use a preset or custom speaker:
```powershell
# Use name (requires corresponding .json file in speakers/ directory)
cargo run --bin qwen3_tts -- --text "The weather is nice today." --speaker dylan
# Use specific file path
cargo run --bin qwen3_tts -- --text "I am a custom voice." --voice-file "path/to/my_voice.json"
```
### 3. Clone New Voice
Clone a voice with just 3-10 seconds of reference audio:
```powershell
cargo run --bin qwen3_tts -- `
--ref-audio "ref.wav" `
--ref-text "The text content corresponding to the reference audio" `
--save-voice "speakers/my_voice.json" `
--text "New voice saved, you can use it directly now!"
```
### 4. Style/Emotion Control (Instruction)
Change the speaking style in real-time via the `--instruction` argument:
```powershell
# Sad tone
cargo run --bin qwen3_tts -- --text "I'm sorry, I didn't mean to..." --instruction "Sad and sobbing, very upset"
# Happy tone
cargo run --bin qwen3_tts -- --text "Awesome! We did it!" --instruction "Happy and excited, slightly faster"
```
### 5. Advanced Configuration
```powershell
cargo run --bin qwen3_tts -- `
--text "Long text generation test." `
--max-steps 1024 ` # Adjust max generation length
--output "output.wav" # Specify output filename
```
### 6. Sampling Parameters
Control randomness and diversity of generation through sampling parameters:
```powershell
# Adjust temperature (higher = more random, 0.0 = greedy)
cargo run --bin qwen3_tts -- --text "Test text" --temperature 0.8
# Use fixed seed for reproducibility
cargo run --bin qwen3_tts -- --text "Test text" --seed 12345
# Full parameter example
cargo run --bin qwen3_tts -- `
--text "Testing sampling parameters." `
--temperature 0.7 ` # Temperature (default 0.7)
--top-k 40 ` # Top-K sampling (default 40)
--top-p 0.9 ` # Top-P nucleus sampling (default 0.9)
--seed 42 # Random seed (optional)
```
## 📂 Directory Structure
The system automatically builds the following structure on first run:
```text
.
├── models/ # Model files (GGUF, ONNX, Tokenizer)
├── runtime/ # Auto-downloaded dependencies (dll, so, dylib)
└── speakers/ # User custom voices
```
## 📜 License & Acknowledgements
- Based on **MIT / Apache 2.0** license.
- Thanks to [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) official repository for models and technical foundation.
- Thanks to [Qwen3-TTS-GGUF](https://github.com/HaujetZhao/Qwen3-TTS-GGUF) for inspiration on the GGUF inference flow.
Generated
+61
View File
@@ -0,0 +1,61 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1779560665,
"narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"utils": "utils"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}
+59
View File
@@ -0,0 +1,59 @@
{
description = "A Nix-flake-based Rust development environment";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
utils.url = "github:numtide/flake-utils";
};
outputs =
{
self,
nixpkgs,
utils,
}:
utils.lib.eachDefaultSystem (
system:
let
pkgs = import nixpkgs { inherit system; };
in
{
devShells.default = pkgs.mkShell {
# 编译工具
nativeBuildInputs = with pkgs; [
pkg-config
rustc
cargo
cargo-watch
cargo-edit
cargo-outdated
];
# 运行时动态链接库
buildInputs = with pkgs; [
openssl
alsa-lib
onnxruntime # <- 新增:NixOS 官方封装的 ONNX Runtime
];
shellHook = ''
# 1. 告诉 ort 库:我已经有系统级的 onnxruntime 了,不要去在线下载
export ORT_STRATEGY="system"
# 2. 让 rust-analyzer / rustc 能找到标准库源码
export RUST_SRC_PATH="${pkgs.rustPlatform.rustLibSrc}"
# 3. 指定 onnxruntime 的动态库和头文件路径(供编译时链接)
export ORT_LIB_DIR="${pkgs.onnxruntime}/lib"
export ORT_INCLUDE_DIR="${pkgs.onnxruntime}/include"
# 4. 运行时让程序能通过 LD_LIBRARY_PATH 找到 .so 文件
export LD_LIBRARY_PATH="${pkgs.onnxruntime}/lib:${pkgs.openssl}/lib:${pkgs.alsa-lib}/lib:$LD_LIBRARY_PATH"
echo "🦀 Rust + ALSA + ONNX 环境已就绪!"
'';
};
}
);
}
+38
View File
@@ -0,0 +1,38 @@
## Summary
修复音频生成断续、缺失问题 (Issue #2)
### 修改内容
1. **随机种子问题修复**
- 将固定的随机种子 `12345` 改为动态生成(使用系统时间 + 熵源)
- 支持用户通过 CLI 参数 `--seed` 指定种子以实现可复现性
2. **解码器边界处理修复**
- 将 codes 截断为 16 的倍数(一帧 = 16 个 codes)
- 使用 `clamp(0, 2047)` 替代 `min(2047)` 确保值在有效范围内
- 保留未满一帧的剩余 codes 供下次迭代处理
3. **采样参数暴露**
- 新增 `SamplerConfig` 结构体
- CLI 新增参数:`--temperature`, `--top_k`, `--top_p`, `--seed`
- 支持运行时调整采样策略
4. **代码质量**
- 修复所有 clippy 警告
- 消除编译警告
### 测试建议
```bash
# 使用默认参数
.\qwen3_tts.exe --text "测试文本" --output "test.wav" --speaker sohee
# 调整温度(更高温度 = 更随机)
.\qwen3_tts.exe --text "测试文本" --output "test.wav" --speaker sohee --temperature 0.7
# 使用固定种子复现结果
.\qwen3_tts.exe --text "测试文本" --output "test.wav" --speaker sohee --seed 12345
```
Closes #2
+2055
View File
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
+49
View File
@@ -0,0 +1,49 @@
{
"speakers": [
{
"name": "vivian",
"id": 3065,
"description": "female_adult"
},
{
"name": "serena",
"id": 3066,
"description": "female_adult"
},
{
"name": "uncle_fu",
"id": 3010,
"description": "male_middle"
},
{
"name": "ryan",
"id": 3061,
"description": "male_young"
},
{
"name": "aiden",
"id": 2861,
"description": "male_young"
},
{
"name": "ono_anna",
"id": 2873,
"description": "female_adult"
},
{
"name": "sohee",
"id": 2864,
"description": "female_young"
},
{
"name": "eric",
"id": 2875,
"description": "male_adult"
},
{
"name": "dylan",
"id": 2878,
"description": "male_adult"
}
]
}
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+2055
View File
File diff suppressed because it is too large. Load diff
+461
View File
@@ -0,0 +1,461 @@
//! Assets for Qwen3-TTS - loads numpy files and provides projections
use std::io::{Read, Seek};
pub struct Assets {
pub tts_pad: Vec<f32>,
pub proj_weight: Vec<f32>,
pub proj_bias: Vec<f32>,
pub codec_embeddings: Vec<Vec<f32>>,
pub text_table: Vec<f32>,
}
impl Assets {
pub fn load(model_dir: &std::path::Path) -> Result<Self, Box<dyn std::error::Error>> {
println!("Loading assets from: {}", model_dir.display());
let gguf_path = model_dir.join("qwen3_assets.gguf");
if gguf_path.exists() {
println!(" Found GGUF assets: {}", gguf_path.display());
return Self::load_gguf(&gguf_path);
}
println!(" GGUF not found, falling back to NPY...");
// Fallback to legacy NPY loading
Self::load_npy(model_dir)
}
fn load_gguf(path: &std::path::Path) -> Result<Self, Box<dyn std::error::Error>> {
// Custom minimal GGUF reader to avoid dependencies
let file = std::fs::File::open(path)?;
let mut reader = std::io::BufReader::new(file);
// 1. Header
let mut magic = [0u8; 4];
reader.read_exact(&mut magic)?;
if &magic != b"GGUF" {
return Err("Not a GGUF file".into());
}
let mut version_bytes = [0u8; 4];
reader.read_exact(&mut version_bytes)?;
let version = u32::from_le_bytes(version_bytes);
if version < 2 {
return Err(format!("Unsupported GGUF version: {}", version).into());
}
let mut tensor_count_bytes = [0u8; 8];
reader.read_exact(&mut tensor_count_bytes)?;
let tensor_count = u64::from_le_bytes(tensor_count_bytes);
let mut kv_count_bytes = [0u8; 8];
reader.read_exact(&mut kv_count_bytes)?;
let kv_count = u64::from_le_bytes(kv_count_bytes);
// Sub-helper to read string
let read_string = |r: &mut std::io::BufReader<std::fs::File>| -> Result<String, Box<dyn std::error::Error>> {
let mut len_bytes = [0u8; 8];
r.read_exact(&mut len_bytes)?;
let len = u64::from_le_bytes(len_bytes) as usize;
let mut bytes = vec![0u8; len];
r.read_exact(&mut bytes)?;
Ok(String::from_utf8(bytes)?)
};
// 2. Metadata KV (Skip)
for _ in 0..kv_count {
let _key = read_string(&mut reader)?;
let mut type_bytes = [0u8; 4];
reader.read_exact(&mut type_bytes)?;
let val_type = u32::from_le_bytes(type_bytes);
match val_type {
0..=7 => {
// Simplified skipping logic
let size = match val_type {
0 | 1 | 7 => 1,
2 | 3 => 2,
4..=6 => 4,
_ => 0,
};
if size > 0 {
let mut b = vec![0u8; size];
reader.read_exact(&mut b)?;
}
}
8 => {
let _s = read_string(&mut reader)?;
} // String
10..=12 => {
let mut b = [0u8; 8];
reader.read_exact(&mut b)?;
} // 64-bit
9 => {
return Err(
"Array type in GGUF metadata not implemented in simple reader".into(),
);
}
_ => return Err(format!("Unknown GGUF value type: {}", val_type).into()),
}
}
// 3. Tensors Info
struct TensorInfo {
name: String,
offset: u64,
shape: Vec<usize>,
_type: u32,
}
let mut tensors = std::collections::HashMap::new();
for _ in 0..tensor_count {
let name = read_string(&mut reader)?;
let mut ndim_b = [0u8; 4];
reader.read_exact(&mut ndim_b)?;
let n_dims = u32::from_le_bytes(ndim_b) as usize;
let mut shape = Vec::new();
for _ in 0..n_dims {
let mut dim_b = [0u8; 8];
reader.read_exact(&mut dim_b)?;
shape.push(u64::from_le_bytes(dim_b) as usize);
}
let mut type_b = [0u8; 4];
reader.read_exact(&mut type_b)?;
let type_id = u32::from_le_bytes(type_b);
let mut offset_b = [0u8; 8];
reader.read_exact(&mut offset_b)?;
let offset = u64::from_le_bytes(offset_b);
tensors.insert(
name.clone(),
TensorInfo {
name,
offset,
shape,
_type: type_id,
},
);
}
// Padding alignment
let current_pos = reader.stream_position()?;
let alignment = 32;
let padding = (alignment - (current_pos % alignment)) % alignment;
reader.seek(std::io::SeekFrom::Current(padding as i64))?;
let data_start = reader.stream_position()?;
let path_clone = path.to_path_buf();
let read_tensor_data = |info: &TensorInfo| -> Result<Vec<f32>, Box<dyn std::error::Error>> {
let mut f = std::fs::File::open(&path_clone)?;
f.seek(std::io::SeekFrom::Start(data_start + info.offset))?;
let num_elems: usize = info.shape.iter().product();
println!(
" Debug: Reading tensor '{}' shape={:?}, elems={}, bytes={}",
info.name,
info.shape,
num_elems,
num_elems * 4
);
if info._type != 0 {
return Err(
format!("Unsupported tensor type: {} (expected F32)", info._type).into(),
);
}
if num_elems * 4 != info.shape.iter().product::<usize>() * 4 {
// Paranoia check
}
// Allocate Vec<f32> directly
let mut floats = Vec::new();
if let Err(e) = floats.try_reserve_exact(num_elems) {
return Err(format!(
"Failed to reserve memory for tensor '{}' (f32): {}",
info.name, e
)
.into());
}
floats.resize(num_elems, 0.0);
// Read directly into the f32 buffer as bytes
// SAFETY: f32 and u8 have same layout size ratio (1:4).
// We just treat the slice of f32s as a mutable slice of u8s.
let byte_slice = unsafe {
std::slice::from_raw_parts_mut(floats.as_mut_ptr() as *mut u8, num_elems * 4)
};
// Read in chunks
let chunk_size = 100 * 1024 * 1024;
let total_bytes = num_elems * 4;
let mut current_read = 0;
while current_read < total_bytes {
let remaining = total_bytes - current_read;
let to_read = remaining.min(chunk_size);
f.read_exact(&mut byte_slice[current_read..current_read + to_read])?;
current_read += to_read;
}
println!(
" Debug: Tensor '{}' read complete (optimized).",
info.name
);
Ok(floats)
};
// Load specific tensors
let proj_weight = read_tensor_data(
tensors
.get("proj.weight")
.ok_or("proj.weight (tensor) missing")?,
)?;
println!(" Debug: proj.weight loaded.");
let proj_bias = read_tensor_data(
tensors
.get("proj.bias")
.ok_or("proj.bias (tensor) missing")?,
)?;
println!(" Debug: proj.bias loaded.");
let text_table = if let Some(info) = tensors.get("text_embd") {
let data = read_tensor_data(info)?;
println!(" Debug: text_embd loaded.");
data
} else {
println!(" WARNING: text_embd not found in GGUF!");
Vec::new()
};
let mut codec_embeddings = Vec::new();
for i in 0..16 {
let name = format!("codec_embd.{}", i);
if let Some(info) = tensors.get(&name) {
codec_embeddings.push(read_tensor_data(info)?);
}
}
println!(" Debug: codec_embeddings loaded.");
let tts_pad = if text_table.len() >= (151671 + 1) * 2048 {
let start = 151671 * 2048;
text_table[start..start + 2048].to_vec()
} else {
vec![0.0; 2048]
};
println!(" Debug: tts_pad loaded.");
println!(
" Loaded via GGUF (Custom Reader): ProjW={}, TextTbl={}, Codec={}",
proj_weight.len(),
text_table.len(),
codec_embeddings.len()
);
Ok(Self {
tts_pad,
proj_weight,
proj_bias,
codec_embeddings,
text_table,
})
}
fn load_npy(model_dir: &std::path::Path) -> Result<Self, Box<dyn std::error::Error>> {
let proj_weight = Self::load_npy_f32(&model_dir.join("proj_weight.npy"))?;
let proj_bias = Self::load_npy_f32_1d(&model_dir.join("proj_bias.npy"))?;
let text_table_path = model_dir.join("text_embedding_projected.npy");
let text_table = if text_table_path.exists() {
Self::load_npy_f32(&text_table_path)?
} else {
Vec::new()
};
let mut codec_embeddings = Vec::new();
for i in 0..16 {
let emb_path = model_dir.join(format!("codec_embedding_{}.npy", i));
if emb_path.exists() {
codec_embeddings.push(Self::load_npy_f32(&emb_path)?);
}
}
let tts_pad = if text_table.len() >= (151671 + 1) * 2048 {
let start = 151671 * 2048;
text_table[start..start + 2048].to_vec()
} else {
vec![0.0; 2048]
};
Ok(Self {
tts_pad,
proj_weight,
proj_bias,
codec_embeddings,
text_table,
})
}
fn load_npy_f32(path: &std::path::Path) -> Result<Vec<f32>, Box<dyn std::error::Error>> {
let mut file = std::fs::File::open(path)?;
let mut magic = [0u8; 10];
file.read_exact(&mut magic)?;
if &magic[..6] != b"\x93NUMPY" {
return Err("Not a numpy file".into());
}
let version = (magic[6], magic[7]);
let header_len = match version {
(1, _) => {
file.seek(std::io::SeekFrom::Start(8))?;
let mut len = [0u8; 2];
file.read_exact(&mut len)?;
u16::from_le_bytes(len) as usize
}
(2, _) => {
file.seek(std::io::SeekFrom::Start(8))?;
let mut len = [0u8; 4];
file.read_exact(&mut len)?;
u32::from_le_bytes(len) as usize
}
_ => return Err("Unsupported numpy version".into()),
};
let mut header = vec![0u8; header_len];
file.read_exact(&mut header)?;
let shape_start = header
.windows(6)
.position(|w| std::str::from_utf8(w).unwrap_or("").starts_with("shape"));
let shape_str = if let Some(pos) = shape_start {
let mut start = pos + 6;
while start < header.len() && header[start] != b'(' {
start += 1;
}
if start < header.len() {
let mut end = start + 1;
while end < header.len() && header[end] != b')' {
end += 1;
}
String::from_utf8_lossy(&header[start + 1..end]).to_string()
} else {
String::new()
}
} else {
String::new()
};
let shape: Vec<usize> = shape_str
.split(',')
.filter(|s| !s.trim().is_empty())
.filter_map(|s| {
s.trim()
.trim_matches(|c| c == '(' || c == ')' || c == '[' || c == ']')
.parse()
.ok()
})
.collect();
let total_elements: usize = shape.iter().product();
let elem_size = 4;
let mut data = vec![0u8; total_elements * elem_size];
file.read_exact(&mut data)?;
let fdata: Vec<f32> = data
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
Ok(fdata)
}
fn load_npy_f32_1d(path: &std::path::Path) -> Result<Vec<f32>, Box<dyn std::error::Error>> {
Self::load_npy_f32(path)
}
pub fn project(&self, hidden: &[f32]) -> Vec<f32> {
let n_in = hidden.len();
let n_out = self.proj_bias.len();
let mut result = vec![0.0; n_out];
for (out_idx, res) in result.iter_mut().enumerate().take(n_out) {
let mut sum = self.proj_bias[out_idx];
for (in_idx, h) in hidden.iter().enumerate().take(n_in) {
// PyTorch Linear weights are [out_features, in_features]
// Correct index: out_idx * n_in + in_idx
sum += h * self.proj_weight[out_idx * n_in + in_idx];
}
*res = sum;
}
result
}
pub fn project_embedding(&self, emb_2048: &[f32]) -> Vec<f32> {
let n_in = 2048;
let n_out = self.proj_bias.len();
let mut result = vec![0.0; n_out];
for (out_idx, res) in result.iter_mut().enumerate().take(n_out) {
let mut sum = self.proj_bias[out_idx];
for (in_idx, val) in emb_2048.iter().enumerate().take(n_in) {
// PyTorch Linear weights are [out_features, in_features]
// Correct index: out_idx * n_in + in_idx
sum += val * self.proj_weight[out_idx * n_in + in_idx];
}
*res = sum;
}
result
}
pub fn get_codec_embedding(&self, q: usize, code: i32) -> Vec<f32> {
if q < self.codec_embeddings.len() {
let emb = &self.codec_embeddings[q];
let code = code.max(0) as usize;
let start = code * 2048;
if start + 2048 <= emb.len() {
return emb[start..start + 2048].to_vec();
} else {
eprintln!(
"WARNING: OOB Code Access: q={}, code={}, start={}, len={}",
q,
code,
start,
emb.len()
);
}
}
vec![0.0; 2048]
}
pub fn get_codec_embedding_1024(&self, q: usize, code: i32) -> Vec<f32> {
let emb_2048 = self.get_codec_embedding(q, code);
self.project_embedding(&emb_2048)
}
pub fn get_text_embedding(&self, token_id: usize) -> Vec<f32> {
let start = token_id * 2048;
if start + 2048 <= self.text_table.len() {
self.text_table[start..start + 2048].to_vec()
} else {
// Fallback for OOB tokens (should not happen usually)
self.get_text_embedding_fallback(token_id)
}
}
pub fn get_text_embedding_fallback(&self, token_id: usize) -> Vec<f32> {
let mut result = vec![0.0; 2048];
for i in 0..2048.min(result.len()) {
result[i] = ((token_id * 17 + i) as f32 % 2.0) - 1.0;
}
result
}
}
+173
View File
@@ -0,0 +1,173 @@
use clap::Parser;
use qwen3_tts::SamplerConfig;
use qwen3_tts::TtsEngine;
use qwen3_tts::VoiceFile;
use std::path::PathBuf;
use std::time::Instant;
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// Model directory (contains models/assets)
#[arg(long, default_value = "models")]
model_dir: PathBuf,
/// Helper: GGUF quantization (none, q5_k_m, q8_0). default: none (unquantized/gguf)
#[arg(long, default_value = "none")]
quant: String,
/// Text to generate
#[arg(short, long)]
text: String,
/// Preset voice file (.json)
#[arg(short, long)]
voice_file: Option<PathBuf>,
/// Reference audio file for feature extraction (.wav)
#[arg(long)]
ref_audio: Option<PathBuf>,
/// Reference text for the record audio
#[arg(long)]
ref_text: Option<String>,
/// Path to save the extracted VoiceFile (.json)
#[arg(long)]
save_voice: Option<PathBuf>,
/// Output audio file
#[arg(short, long, default_value = "output.wav")]
output: PathBuf,
/// Maximum generation steps (tokens)
#[arg(long, default_value_t = 512)]
max_steps: usize,
/// Speakers directory
#[arg(long, default_value = "speakers")]
speakers_dir: PathBuf,
/// Speaker name or ID (fallback to vivian)
#[arg(short, long)]
speaker: Option<String>,
/// Instruction style (e.g. "Happy", "Sad") - prepended to text
#[arg(long)]
instruction: Option<String>,
/// Temperature for sampling (higher = more random, 0.0 = greedy)
#[arg(long, default_value_t = 0.7)]
temperature: f32,
/// Top-K sampling (0 = disabled)
#[arg(long, default_value_t = 40)]
top_k: i32,
/// Top-P (nucleus) sampling (1.0 = disabled)
#[arg(long, default_value_t = 0.9)]
top_p: f32,
/// Random seed for reproducibility (omit for random)
#[arg(long)]
seed: Option<u64>,
/// Number of threads for inference
#[arg(long, default_value_t = 4)]
threads: i32,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
let start_total = Instant::now();
println!("=== Qwen3-TTS Example CLI ===");
println!("Model Dir: {:?}", args.model_dir);
println!("Quant: {}", args.quant);
println!("Threads: {}", args.threads);
println!("Voice: {:?}", args.voice_file);
println!("Text: {}", args.text);
// 0. Auto-download (Explicit step)
println!("Checking models...");
TtsEngine::download_models(&args.model_dir, &args.quant)
.await
.map_err(|e| format!("Model download failed: {}", e))?;
// 1. Load Engine
let mut engine = TtsEngine::new(&args.model_dir, &args.quant, args.threads)
.await
.map_err(|e| format!("Engine load failed: {}", e))?;
engine.set_max_steps(args.max_steps);
// 1.1 Configure sampler
let sampler_config = SamplerConfig::new(args.temperature, args.top_k, args.top_p, args.seed);
engine.set_sampler_config(sampler_config);
println!(
"Sampler: temp={}, top_k={}, top_p={}, seed={:?}",
args.temperature, args.top_k, args.top_p, args.seed
);
// 1.5 Load custom speakers if dir specified
if args.speakers_dir.exists() {
engine.load_speakers(&args.speakers_dir)?;
}
// 2. Load or Create Voice File
let voice = if let Some(audio_path) = args.ref_audio {
println!("Creating voice from reference: {:?}", audio_path);
let ref_text = args.ref_text.unwrap_or_default();
let vf = engine
.create_voice_file(audio_path, ref_text)
.map_err(|e| format!("Feature extraction failed: {}", e))?;
if let Some(save_path) = args.save_voice {
vf.save(&save_path)
.map_err(|e| format!("Failed to save voice file: {}", e))?;
println!("Saved new voice file to: {:?}", save_path);
}
vf
} else if let Some(vf_path) = args.voice_file {
println!("Loading voice from file: {:?}", vf_path);
VoiceFile::load(vf_path).map_err(|e| format!("Failed to load voice file: {}", e))?
} else {
let spk_id = args.speaker.as_deref().unwrap_or("vivian");
println!("Using speaker from cache: {}", spk_id);
engine.get_speaker(spk_id).clone()
};
println!("Voice Name: {}", voice.name.as_deref().unwrap_or("Dynamic"));
// 3. Generate
let start_gen = Instant::now();
println!("Generating...");
// Generate
let audio = engine
.generate_with_voice_streaming(&args.text, &voice, args.instruction.as_deref(), None)
.map_err(|e| format!("Generation failed: {}", e))?;
let gen_duration = start_gen.elapsed();
println!("Generation took: {:.2}s", gen_duration.as_secs_f32());
let audio_duration_secs = audio.samples.len() as f32 / audio.sample_rate as f32;
let rtf = gen_duration.as_secs_f32() / audio_duration_secs;
println!(
"Audio: {:.2}s ({} samples @ {} Hz), RTF: {:.3}",
audio_duration_secs,
audio.samples.len(),
audio.sample_rate,
rtf
);
// 4. Save
audio
.save_wav(&args.output)
.map_err(|e| format!("Failed to save output: {}", e))?;
println!("Saved to: {:?}", args.output);
println!("Total time: {:.2}s", start_total.elapsed().as_secs_f32());
Ok(())
}
+366
View File
@@ -0,0 +1,366 @@
use axum::{
extract::{
ws::{Message, WebSocket, WebSocketUpgrade},
DefaultBodyLimit, State,
},
http::header,
response::{Html, IntoResponse},
routing::{get, post},
Json, Router,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
use clap::Parser;
use qwen3_tts::{TtsEngine, VoiceFile};
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, net::SocketAddr, path::PathBuf, sync::Arc, time::Instant};
use tokio::sync::Mutex;
use tower_http::cors::{Any, CorsLayer};
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
#[arg(long, default_value = "models")]
model_dir: PathBuf,
#[arg(long, default_value = "q5_k_m")]
quant: String,
#[arg(long, default_value = "speakers")]
speakers_dir: PathBuf,
#[arg(long, default_value = "3000")]
port: u16,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 4)]
threads: i32,
}
struct AppState {
engine: Arc<Mutex<TtsEngine>>,
speakers: HashMap<String, VoiceFile>,
}
#[derive(Debug, Serialize, Deserialize)]
struct TTSRequest {
text: String,
speaker: Option<String>,
temperature: Option<f32>,
top_k: Option<i32>,
top_p: Option<f32>,
seed: Option<u64>,
instruction: Option<String>,
max_steps: Option<usize>,
}
#[derive(Debug, Serialize, Deserialize)]
struct TTSResponse {
success: bool,
message: Option<String>,
audio_base64: Option<String>,
sample_rate: Option<u32>,
duration_ms: Option<f64>,
}
#[derive(Debug, Serialize, Deserialize)]
struct SpeakersResponse {
speakers: Vec<String>,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
println!("=== Qwen3-TTS Server ===");
println!("Model Dir: {:?}", args.model_dir);
println!("Quant: {}", args.quant);
println!("Port: {}", args.port);
println!("Threads: {}", args.threads);
println!("Checking models...");
TtsEngine::download_models(&args.model_dir, &args.quant)
.await
.map_err(|e| format!("Model download failed: {}", e))?;
println!("Loading engine...");
let mut engine = TtsEngine::new(&args.model_dir, &args.quant, args.threads)
.await
.map_err(|e| format!("Engine load failed: {}", e))?;
if args.speakers_dir.exists() {
engine.load_speakers(&args.speakers_dir)?;
}
let speakers = engine.get_speakers_map().clone();
let engine = Arc::new(Mutex::new(engine));
let state = AppState { engine, speakers };
let app = Router::new()
.route("/", get(index))
.route("/audio-processor.js", get(audio_processor))
.route("/api/tts", post(tts_handler))
.route("/api/tts/stream", get(tts_stream_handler))
.route("/api/speakers", get(speakers_handler))
.route("/health", get(health))
.layer(
CorsLayer::new()
.allow_origin(Any)
.allow_methods(Any)
.allow_headers(Any),
)
.layer(DefaultBodyLimit::max(50 * 1024 * 1024))
.with_state(Arc::new(state));
let addr: SocketAddr = format!("{}:{}", args.host, args.port).parse()?;
println!("Server running at http://{}", addr);
println!("API endpoints:");
println!(" POST /api/tts - Generate TTS audio (non-streaming)");
println!(" GET /api/tts/stream - WebSocket streaming TTS");
println!(" GET /api/speakers - List available speakers");
println!(" GET /health - Health check");
let listener = tokio::net::TcpListener::bind(&addr).await?;
axum::serve(listener, app).await?;
Ok(())
}
async fn index() -> Html<&'static str> {
Html(include_str!("../../webui/index.html"))
}
async fn audio_processor() -> impl IntoResponse {
(
[(header::CONTENT_TYPE, "application/javascript")],
include_str!("../../webui/audio-processor.js"),
)
}
async fn health() -> &'static str {
"OK"
}
async fn speakers_handler(State(state): State<Arc<AppState>>) -> Json<SpeakersResponse> {
let names: Vec<String> = state.speakers.keys().cloned().collect();
Json(SpeakersResponse { speakers: names })
}
async fn tts_handler(
State(state): State<Arc<AppState>>,
Json(req): Json<TTSRequest>,
) -> Json<TTSResponse> {
let start = Instant::now();
let speaker_name = req.speaker.unwrap_or_else(|| "vivian".to_string());
let voice = state.speakers.get(&speaker_name).cloned();
let voice = match voice {
Some(v) => v,
None => {
return Json(TTSResponse {
success: false,
message: Some(format!("Speaker '{}' not found", speaker_name)),
audio_base64: None,
sample_rate: None,
duration_ms: None,
});
}
};
let mut engine = state.engine.lock().await;
// 一次性修改采样器配置
{
let config = engine.get_sampler_config().clone();
let mut new_config = config;
if let Some(temp) = req.temperature {
new_config.temperature = temp;
}
if let Some(top_k) = req.top_k {
new_config.top_k = top_k;
}
if let Some(top_p) = req.top_p {
new_config.top_p = top_p;
}
if let Some(seed) = req.seed {
new_config.seed = Some(seed);
}
engine.set_sampler_config(new_config);
}
// 设置 max_steps
if let Some(max_steps) = req.max_steps {
engine.set_max_steps(max_steps);
}
let result =
engine.generate_with_voice_streaming(&req.text, &voice, req.instruction.as_deref(), None);
match result {
Ok(audio) => {
let duration_ms = (audio.samples.len() as f64 / audio.sample_rate as f64) * 1000.0;
let audio_duration_secs = audio.samples.len() as f64 / audio.sample_rate as f64;
let rtf = start.elapsed().as_secs_f64() / audio_duration_secs;
println!(
"RTF: {:.3} (gen: {:.2}s, audio: {:.2}s)",
rtf,
start.elapsed().as_secs_f64(),
audio_duration_secs
);
let wav_data = audio.to_wav_bytes();
Json(TTSResponse {
success: true,
message: None,
audio_base64: Some(BASE64.encode(&wav_data)),
sample_rate: Some(audio.sample_rate),
duration_ms: Some(duration_ms),
})
}
Err(e) => Json(TTSResponse {
success: false,
message: Some(e),
audio_base64: None,
sample_rate: None,
duration_ms: None,
}),
}
}
// WebSocket 流式 TTS 处理器
async fn tts_stream_handler(
ws: WebSocketUpgrade,
State(state): State<Arc<AppState>>,
) -> axum::response::Response {
ws.on_upgrade(move |socket| handle_tts_stream(socket, state))
}
async fn handle_tts_stream(mut socket: WebSocket, state: Arc<AppState>) {
let (text_tx, text_rx) = std::sync::mpsc::channel::<(String, Option<String>, Option<String>)>();
let (audio_tx, mut audio_rx) = tokio::sync::mpsc::unbounded_channel::<Vec<f32>>();
let engine = state.engine.clone();
let speakers = state.speakers.clone();
let mut total_samples: usize = 0;
let stream_start = std::time::Instant::now();
std::thread::spawn(move || {
let mut engine = engine.blocking_lock();
let mut voice: Option<VoiceFile> = None;
while let Ok((text, speaker_name, instruction)) = text_rx.recv() {
if text.is_empty() {
break;
}
if voice.is_none() {
let spk_name = speaker_name.unwrap_or_else(|| "vivian".to_string());
voice = speakers.get(&spk_name).cloned();
if voice.is_none() {
let _ = audio_tx.send(vec![]);
break;
}
}
let voice_ref = voice.as_ref().unwrap();
let result = engine.generate_with_voice_streaming(
&text,
voice_ref,
instruction.as_deref(),
Some(audio_tx.clone()),
);
if result.is_err() {
break;
}
let _ = audio_tx.send(vec![]);
}
});
let mut current_speaker: Option<String> = None;
let mut end_requested = false;
loop {
if end_requested {
match audio_rx.recv().await {
Some(samples) => {
if samples.is_empty() {
let _ = socket.send(Message::Text("segment_done".to_string())).await;
} else {
let bytes: Vec<u8> = samples.iter().flat_map(|s| s.to_le_bytes()).collect();
if socket.send(Message::Binary(bytes)).await.is_err() {
break;
}
}
}
None => break,
}
} else {
tokio::select! {
Some(samples) = audio_rx.recv() => {
if samples.is_empty() {
let gen_time = stream_start.elapsed().as_secs_f64();
let audio_duration = total_samples as f64 / 24000.0;
let rtf = if audio_duration > 0.0 { gen_time / audio_duration } else { 0.0 };
eprintln!("[Stream] RTF: {:.3} (gen: {:.2}s, audio: {:.2}s)", rtf, gen_time, audio_duration);
let _ = socket.send(Message::Text("segment_done".to_string())).await;
} else {
total_samples += samples.len();
eprintln!("[Stream] received {} samples, total: {}", samples.len(), total_samples);
let bytes: Vec<u8> = samples
.iter()
.flat_map(|s| s.to_le_bytes())
.collect();
if socket.send(Message::Binary(bytes)).await.is_err() {
break;
}
}
}
msg = socket.recv() => {
match msg {
Some(Ok(Message::Text(text))) => {
if text == "end" {
let _ = text_tx.send((String::new(), None, None));
end_requested = true;
} else {
let req: TTSRequest = match serde_json::from_str(&text) {
Ok(r) => r,
Err(e) => {
let _ = socket.send(Message::Text(format!("{{\"error\": \"Invalid request: {}\"}}", e))).await;
continue;
}
};
if current_speaker.is_none() {
current_speaker = req.speaker.clone();
}
let text_to_generate = req.text.trim();
if !text_to_generate.is_empty() {
let _ = text_tx.send((text_to_generate.to_string(), current_speaker.clone(), req.instruction.clone()));
}
}
}
Some(Ok(Message::Close(_))) => break,
Some(Ok(Message::Binary(_))) => continue,
Some(Ok(Message::Ping(_))) => continue,
Some(Ok(Message::Pong(_))) => continue,
Some(Err(_)) => break,
None => break,
}
}
}
}
}
let _ = text_tx.send((String::new(), None, None));
let _ = socket.send(Message::Text("done".to_string())).await;
}
+364
View File
@@ -0,0 +1,364 @@
use futures_util::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use reqwest::Client;
use std::fs::File;
use std::io::Write;
use std::path::Path;
pub struct Downloader {
client: Client,
model_base_url: String,
}
impl Downloader {
pub async fn new() -> Self {
let client = Client::new();
// Check connectivity to huggingface.co for MODELS
let hf_base = if Self::check_connectivity(&client, "https://huggingface.co").await {
"https://huggingface.co"
} else {
"https://hf-mirror.com"
};
let model_base_url = format!("{}/cgisky/qwen3-tts-custom-gguf/resolve/main", hf_base);
Self {
client,
model_base_url,
}
}
async fn check_connectivity(client: &Client, url: &str) -> bool {
let result = client
.head(url)
.timeout(std::time::Duration::from_secs(3))
.send()
.await;
result.is_ok()
}
/// Check and download both models and runtime libraries
pub async fn check_and_download(
&self,
model_dir: &Path,
quant: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// 1. Check Models
self.download_models(model_dir, quant).await?;
// 2. Check Runtime Libraries (DLLs/SOs)
self.download_runtimes().await?;
Ok(())
}
async fn download_models(
&self,
model_dir: &Path,
quant: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let quant_folder = match quant {
"q5_k_m" => "gguf_q5_k_m",
"q8_0" => "gguf_q8_0",
_ => "gguf",
};
let files = vec![
(
"onnx/qwen3_tts_decoder.onnx".to_owned(),
"onnx/qwen3_tts_decoder.onnx".to_owned(),
1_000_000usize,
),
(
"tokenizer/tokenizer.json".to_owned(),
"tokenizer/tokenizer.json".to_owned(),
1_000_000usize,
),
(
format!("{}/qwen3_assets.gguf", quant_folder),
format!("{}/qwen3_assets.gguf", quant_folder),
500_000_000usize,
),
(
format!("{}/qwen3_tts_talker.gguf", quant_folder),
format!("{}/qwen3_tts_talker.gguf", quant_folder),
100_000_000usize,
),
(
format!("{}/qwen3_tts_predictor.gguf", quant_folder),
format!("{}/qwen3_tts_predictor.gguf", quant_folder),
50_000_000usize,
),
];
for (remote, local, min_size) in files {
let local_path = model_dir.join(&local);
let should_download = match std::fs::metadata(&local_path) {
Ok(meta) => meta.len() < min_size as u64,
Err(_) => true,
};
if should_download {
if let Some(p) = local_path.parent() {
std::fs::create_dir_all(p)?;
}
let url = format!("{}/{}", self.model_base_url, remote);
if local_path.exists() {
println!(
"Refreshing Model: {} (found {} bytes, expected at least {})...",
local,
std::fs::metadata(&local_path)?.len(),
min_size
);
let _ = std::fs::remove_file(&local_path);
} else {
println!("Downloading Model: {}...", local);
}
self.download_to_file(&url, &local_path).await?;
}
}
Ok(())
}
pub async fn download_runtimes(&self) -> Result<(), Box<dyn std::error::Error>> {
let runtime_dir = Path::new("runtime");
if !runtime_dir.exists() {
std::fs::create_dir_all(runtime_dir)?;
}
let os = if cfg!(target_os = "windows") {
"win"
} else if cfg!(target_os = "linux") {
"linux"
} else if cfg!(target_os = "macos") {
"osx"
} else {
return Err("Unsupported OS for auto-runtime download".into());
};
let arch = if cfg!(target_arch = "x86_64") {
"x64"
} else if cfg!(target_arch = "aarch64") {
"arm64"
} else {
return Err("Unsupported Architecture for auto-runtime download".into());
};
// Determine Backend
let backend = if cfg!(feature = "cuda") {
"cuda"
} else if cfg!(target_os = "macos") {
"" // macOS uses Metal integrated in the binary
} else {
"vulkan"
};
// Platform mapping for llama.cpp release assets
// Windows -> win, Linux -> ubuntu, macOS -> macos
let release_os = if cfg!(target_os = "windows") {
"win"
} else if cfg!(target_os = "macos") {
"macos"
} else {
"ubuntu"
};
let release_ext = if cfg!(target_os = "windows") {
"zip"
} else {
"tar.gz"
};
// 1. ONNX Runtime
let ort_version = "1.23.2";
let ort_ext = if os == "win" { "zip" } else { "tgz" };
let ort_filename = format!("onnxruntime-{}-{}-{}.{}", os, arch, ort_version, ort_ext);
let ort_dll_name = if os == "win" {
"onnxruntime.dll"
} else if os == "osx" {
"libonnxruntime.dylib"
} else {
"libonnxruntime.so"
};
if !runtime_dir.join(ort_dll_name).exists() {
println!("Downloading ONNX Runtime ({}-{})...", os, arch);
let ort_url = format!(
"https://github.com/microsoft/onnxruntime/releases/download/v{}/{}",
ort_version, ort_filename
);
let tmp_zip = Path::new("ort_runtime.tmp");
self.download_to_file(&ort_url, tmp_zip).await?;
println!("Extracting ONNX Runtime...");
if ort_ext == "zip" {
self.extract_zip(
tmp_zip,
runtime_dir,
&format!("onnxruntime-{}-{}-{}", os, arch, ort_version),
"lib",
)?;
} else {
self.extract_targz(
tmp_zip,
runtime_dir,
&format!("onnxruntime-{}-{}-{}", os, arch, ort_version),
"lib",
)?;
}
let _ = std::fs::remove_file(tmp_zip);
}
// 2. Llama.cpp / GGML (Official Release)
let llama_dll = if os == "win" {
"llama.dll"
} else if os == "osx" {
"libllama.dylib"
} else {
"libllama.so"
};
if !runtime_dir.join(llama_dll).exists() {
println!(
"Downloading Llama.cpp Runtimes (Official {} vB7885)...",
if backend.is_empty() { "Metal" } else { backend }
);
// URL Patterns:
// win: llama-b7885-bin-win-vulkan-x64.zip
// linux: llama-b7885-bin-ubuntu-vulkan-x64.tar.gz
// macos: llama-b7885-bin-macos-arm64.tar.gz
let asset_name = if backend.is_empty() {
format!("llama-b7885-bin-{}-{}.{}", release_os, arch, release_ext)
} else {
format!(
"llama-b7885-bin-{}-{}-{}.{}",
release_os, backend, arch, release_ext
)
};
let llama_url = format!(
"https://github.com/ggerganov/llama.cpp/releases/download/b7885/{}",
asset_name
);
let tmp_zip = Path::new("llama_runtime.tmp");
if let Err(e) = self.download_to_file(&llama_url, tmp_zip).await {
eprintln!(
"Warning: Failed to download llama runtime: {}. Please install manually.",
e
);
} else {
if release_ext == "zip" {
self.extract_zip(tmp_zip, runtime_dir, "", "")?;
} else {
self.extract_targz(tmp_zip, runtime_dir, "", "")?;
}
let _ = std::fs::remove_file(tmp_zip);
}
}
Ok(())
}
async fn download_to_file(
&self,
url: &str,
path: &Path,
) -> Result<(), Box<dyn std::error::Error>> {
let res = self.client.get(url).send().await?;
if !res.status().is_success() {
return Err(format!("HTTP {} for {}", res.status(), url).into());
}
let total = res.content_length().unwrap_or(0);
let pb = ProgressBar::new(total);
pb.set_style(ProgressStyle::default_bar()
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})")?
.progress_chars("#>-"));
let mut file = File::create(path)?;
let mut stream = res.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk?;
file.write_all(&chunk)?;
pb.inc(chunk.len() as u64);
}
pb.finish();
Ok(())
}
fn extract_zip(
&self,
zip_path: &Path,
dest: &Path,
prefix: &str,
lib_subdir: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let file = File::open(zip_path)?;
let mut archive = zip::ZipArchive::new(file)?;
for i in 0..archive.len() {
let mut file = archive.by_index(i)?;
let outpath = match file.enclosed_name() {
Some(path) => path.to_owned(),
None => continue,
};
// Filter by prefix and subdirectory if provided
let path_str = outpath.to_string_lossy();
if !prefix.is_empty() && !path_str.starts_with(prefix) {
continue;
}
if !lib_subdir.is_empty() && !path_str.contains(lib_subdir) {
// For ONNX, we only want the DLLs in the lib/ folder
if !path_str.ends_with(".dll")
&& !path_str.ends_with(".so")
&& !path_str.ends_with(".dylib")
{
continue;
}
}
let file_name = outpath.file_name().unwrap();
let dest_file = dest.join(file_name);
if file.is_dir() {
std::fs::create_dir_all(&dest_file)?;
} else {
let mut outfile = File::create(&dest_file)?;
std::io::copy(&mut file, &mut outfile)?;
}
}
Ok(())
}
fn extract_targz(
&self,
tar_path: &Path,
dest: &Path,
prefix: &str,
_lib_subdir: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let tar_gz = File::open(tar_path)?;
let tar = flate2::read::GzDecoder::new(tar_gz);
let mut archive = tar::Archive::new(tar);
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?.into_owned();
let path_str = path.to_string_lossy();
if !prefix.is_empty() && !path_str.starts_with(prefix) {
continue;
}
// Simplified check for libraries
if path_str.ends_with(".so")
|| path_str.ends_with(".dylib")
|| path_str.ends_with(".dll")
{
let file_name = path.file_name().unwrap();
let dest_file = dest.join(file_name);
entry.unpack(dest_file)?;
}
}
Ok(())
}
}
+20
View File
@@ -0,0 +1,20 @@
//! Qwen3-TTS Rust Library
//! Optimized and organized structure for open source.
pub mod assets_manager;
pub mod download;
pub mod models;
pub mod tts;
pub mod utils;
// Re-export core types for convenience
pub use tts::engine::SamplerConfig;
pub use tts::engine::TtsEngine;
pub use tts::prompt::PromptBuilder;
pub use utils::audio::AudioSample;
pub use utils::tokenizer::Tokenizer;
pub use utils::voice_file::VoiceFile;
pub fn cleanup() {
models::llama::cleanup();
}
File diff suppressed because it is too large. Load diff
+2
View File
@@ -0,0 +1,2 @@
pub mod llama;
pub mod onnx;
+559
View File
@@ -0,0 +1,559 @@
//! 真正的 ONNX 模型推理实现
//! 使用 ort crate 进行 ONNX Runtime 推理
//! 执行提供者: DirectML (Windows GPU)
use ndarray::{Array3, Array4};
use ort::session::builder::GraphOptimizationLevel;
use ort::session::Session;
use ort::session::SessionInputValue;
use ort::value::Tensor;
use std::error::Error;
/// 创建 CPU Session (fallback)
fn create_cpu_session(model_path: &str) -> Result<Session, Box<dyn Error>> {
println!(" [ONNX] Requesting Execution Providers: CPU");
let builder = Session::builder()?;
println!(" [ONNX] CPU Session Builder created.");
let builder = builder.with_optimization_level(GraphOptimizationLevel::Level3)?;
let cpu = ort::execution_providers::CPUExecutionProvider::default().build();
let mut builder = builder.with_execution_providers([cpu])?;
println!(" [ONNX] CPU Provider configured.");
let session = builder.commit_from_file(model_path)?;
println!(" [ONNX] CPU Session committed.");
Ok(session)
}
/// 创建 GPU Session (使用 DirectML)
#[cfg(windows)]
fn create_gpu_session(model_path: &str) -> Result<Session, Box<dyn Error>> {
println!(" [ONNX] Requesting Execution Providers: DirectML");
let builder = Session::builder()?;
let builder = builder.with_optimization_level(GraphOptimizationLevel::Level3)?;
let dml = DirectMLExecutionProvider::default().build();
match builder.with_execution_providers([dml]) {
Ok(builder) => {
println!(" [ONNX] DirectML Provider configured.");
let session = builder.commit_from_file(model_path)?;
println!(" [ONNX] DirectML Session committed.");
Ok(session)
}
Err(e) => {
println!(" [ONNX] DirectML failed: {:?}, falling back to CPU.", e);
create_cpu_session(model_path)
}
}
}
/// 打印当前使用的执行提供者信息
fn print_session_info(session: &Session, name: &str) {
let inputs: Vec<_> = session
.inputs()
.iter()
.map(|i| i.name().to_string())
.collect();
let outputs: Vec<_> = session
.outputs()
.iter()
.map(|o| o.name().to_string())
.collect();
println!(" Inputs: {:?}", inputs);
println!(" Outputs: {:?}", outputs);
println!(" [ONNX] {}: Loaded successfully", name);
}
/// 音频编码器 - 将 24kHz 音频编码为 codec codes
pub struct AudioEncoder {
session: Session,
}
impl AudioEncoder {
pub fn load(model_path: &str) -> Result<Self, Box<dyn Error>> {
println!(" [ONNX] AudioEncoder: Loading {}", model_path);
// Windows 上尝试使用 DirectML/CUDA 加速
#[cfg(windows)]
let session = create_gpu_session(model_path)?;
#[cfg(not(windows))]
let session = create_cpu_session(model_path)?;
print_session_info(&session, "AudioEncoder");
Ok(AudioEncoder { session })
}
pub fn encode(&mut self, audio: &[f32]) -> Result<Vec<i64>, Box<dyn Error>> {
println!(" [ONNX] AudioEncoder: Encoding {} samples", audio.len());
// 使用 (shape, data) 形式创建 Tensor
let shape = vec![1i64, audio.len() as i64];
let input_tensor = Tensor::from_array((shape, audio.to_vec()))?;
let outputs = self
.session
.run(ort::inputs!["input_values" => input_tensor])?;
// 输出: audio_codes [1, T//2000, 16]
let codes_output = &outputs["audio_codes"];
let codes_raw = codes_output.try_extract_tensor::<i64>()?;
println!(" [ONNX] AudioEncoder: Output Shape: {:?}", codes_raw.0);
let codes: Vec<i64> = codes_raw.1.to_vec();
let n_frames = codes.len() / 16;
println!(
" [ONNX] AudioEncoder: Encoded {} frames, {} codes",
n_frames,
codes.len()
);
Ok(codes)
}
}
/// 说话人编码器 - 提取说话人嵌入
pub struct SpeakerEncoder {
session: Session,
}
impl SpeakerEncoder {
pub fn load(model_path: &str) -> Result<Self, Box<dyn Error>> {
println!(" [ONNX] SpeakerEncoder: Loading {}", model_path);
// Windows 上尝试使用 DirectML/CUDA 加速
#[cfg(windows)]
let session = create_gpu_session(model_path)?;
#[cfg(not(windows))]
let session = create_cpu_session(model_path)?;
print_session_info(&session, "SpeakerEncoder");
Ok(SpeakerEncoder { session })
}
pub fn encode(&mut self, audio: &[f32]) -> Result<Vec<f32>, Box<dyn Error>> {
println!(
" [ONNX] SpeakerEncoder: Extracting from {} samples",
audio.len()
);
// 计算 Mel 谱图 (简化版)
let (n_frames, mels_data) = self.compute_mel(audio);
let shape = vec![1i64, n_frames as i64, 128i64];
let mels_tensor = Tensor::from_array((shape, mels_data))?;
let outputs = self.session.run(ort::inputs!["mels" => mels_tensor])?;
// 输出: spk_emb [1, 2048]
let emb_output = &outputs["spk_emb"];
let emb_raw = emb_output.try_extract_tensor::<f32>()?;
let emb: Vec<f32> = emb_raw.1.to_vec();
println!(
" [ONNX] SpeakerEncoder: Extracted {} dims embedding",
emb.len()
);
Ok(emb)
}
/// 计算 Mel 谱图 (手动实现,完全对齐 Python librosa)
/// 参数: 24kHz, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000
fn compute_mel(&self, audio: &[f32]) -> (usize, Vec<f32>) {
use rustfft::{num_complex::Complex, FftPlanner};
const SAMPLE_RATE: f32 = 24000.0;
const N_FFT: usize = 1024;
const HOP_LENGTH: usize = 256;
const N_MELS: usize = 128;
const FMIN: f32 = 0.0;
const FMAX: f32 = 12000.0;
// ==================== 1. Mel Filterbank (Slaney style) ====================
// Hz to Mel (Slaney formula)
fn hz_to_mel(freq: f32) -> f32 {
let f_min = 0.0f32;
let f_sp = 200.0 / 3.0;
let min_log_hz = 1000.0f32;
let min_log_mel = (min_log_hz - f_min) / f_sp;
let logstep = 6.4f32.ln() / 27.0;
if freq >= min_log_hz {
min_log_mel + ((freq / min_log_hz).ln() / logstep)
} else {
(freq - f_min) / f_sp
}
}
// Mel to Hz (Slaney formula)
fn mel_to_hz(mel: f32) -> f32 {
let f_min = 0.0f32;
let f_sp = 200.0 / 3.0;
let min_log_hz = 1000.0f32;
let min_log_mel = (min_log_hz - f_min) / f_sp;
let logstep = 6.4f32.ln() / 27.0;
if mel >= min_log_mel {
min_log_hz * (logstep * (mel - min_log_mel)).exp()
} else {
f_min + f_sp * mel
}
}
// Create Mel filterbank (Slaney normalization)
let n_fft_bins = N_FFT / 2 + 1;
let mel_min = hz_to_mel(FMIN);
let mel_max = hz_to_mel(FMAX);
// Mel bin edges
let mut mel_edges = Vec::with_capacity(N_MELS + 2);
for i in 0..=(N_MELS + 1) {
let mel = mel_min + (mel_max - mel_min) * (i as f32) / ((N_MELS + 1) as f32);
mel_edges.push(mel_to_hz(mel));
}
// FFT bin frequencies
let fft_freqs: Vec<f32> = (0..n_fft_bins)
.map(|i| (i as f32) * SAMPLE_RATE / (N_FFT as f32))
.collect();
// Build filterbank matrix [n_mels, n_fft_bins]
let mut mel_filterbank = vec![0.0f32; N_MELS * n_fft_bins];
for m in 0..N_MELS {
let f_left = mel_edges[m];
let f_center = mel_edges[m + 1];
let f_right = mel_edges[m + 2];
// Slaney-style normalization: 2.0 / (f_right - f_left)
let norm = 2.0 / (f_right - f_left);
for (k, &freq) in fft_freqs.iter().enumerate() {
let weight = if freq >= f_left && freq <= f_center {
(freq - f_left) / (f_center - f_left)
} else if freq > f_center && freq <= f_right {
(f_right - freq) / (f_right - f_center)
} else {
0.0
};
mel_filterbank[m * n_fft_bins + k] = weight * norm;
}
}
// ==================== 2. STFT ====================
// Reflect padding
let padding = (N_FFT - HOP_LENGTH) / 2;
let mut padded: Vec<f32> = Vec::with_capacity(padding + audio.len() + padding);
// Reflect at start
for i in (1..=padding).rev() {
if i < audio.len() {
padded.push(audio[i]);
} else {
padded.push(0.0);
}
}
padded.extend_from_slice(audio);
// Reflect at end
for i in 1..=padding {
let idx = audio.len().saturating_sub(1 + i);
if idx < audio.len() {
padded.push(audio[idx]);
} else {
padded.push(0.0);
}
}
// Hann window
let hann_window: Vec<f32> = (0..N_FFT)
.map(|i| 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / N_FFT as f32).cos()))
.collect();
// FFT planner
let mut planner = FftPlanner::<f32>::new();
let fft = planner.plan_fft_forward(N_FFT);
// Process frames
let n_frames = (padded.len().saturating_sub(N_FFT)) / HOP_LENGTH + 1;
let mut all_mels = Vec::with_capacity(n_frames * N_MELS);
for frame_idx in 0..n_frames {
let start = frame_idx * HOP_LENGTH;
if start + N_FFT > padded.len() {
break;
}
// Apply window and convert to complex
let mut fft_buffer: Vec<Complex<f32>> = (0..N_FFT)
.map(|i| Complex::new(padded[start + i] * hann_window[i], 0.0))
.collect();
// Perform FFT
fft.process(&mut fft_buffer);
// Compute magnitudes (add 1e-9 for numerical stability, like Python)
let magnitudes: Vec<f32> = fft_buffer[..n_fft_bins]
.iter()
.map(|c| (c.norm_sqr() + 1e-9).sqrt())
.collect();
// Apply Mel filterbank
for m in 0..N_MELS {
let mut mel_val = 0.0f32;
for k in 0..n_fft_bins {
mel_val += mel_filterbank[m * n_fft_bins + k] * magnitudes[k];
}
// Log compression: log(max(mel, 1e-5))
let log_mel = mel_val.max(1e-5).ln();
all_mels.push(log_mel);
}
}
let actual_n_frames = all_mels.len() / N_MELS;
(actual_n_frames, all_mels)
}
}
/// 音频解码器 - 将 codec codes 解码为音频波形
pub struct AudioDecoder {
session: Session,
}
impl AudioDecoder {
pub fn load(model_path: &str) -> Result<Self, Box<dyn Error>> {
println!(" [ONNX] AudioDecoder: Loading {}", model_path);
// Windows 上尝试使用 DirectML 加速
#[cfg(windows)]
let session = create_gpu_session(model_path)?;
#[cfg(not(windows))]
let session = create_cpu_session(model_path)?;
print_session_info(&session, "AudioDecoder");
Ok(AudioDecoder { session })
}
pub fn create_state() -> DecoderState {
DecoderState::new()
}
pub fn decode(
&mut self,
codes: &[i64],
state: &mut DecoderState,
is_final: bool,
) -> Result<Vec<f32>, Box<dyn Error>> {
let n_frames = codes.len() / 16;
// 如果没有帧且不是 is_final,直接返回空
if n_frames == 0 && !is_final {
return Ok(vec![]);
}
// 1. Prepare Inputs
let mut inputs_vec: Vec<(std::borrow::Cow<'_, str>, SessionInputValue<'_>)> = Vec::new();
// audio_codes: [1, N, 16] (N can be 0 for flushing)
let codes_shape = vec![1i64, n_frames as i64, 16i64];
let codes_tensor = Tensor::from_array((codes_shape, codes.to_vec()))?;
inputs_vec.push(("audio_codes".into(), codes_tensor.into()));
// is_last: [1]
let is_last_val = if is_final { 1.0f32 } else { 0.0f32 };
let is_last_tensor = Tensor::from_array((vec![1i64], vec![is_last_val]))?;
inputs_vec.push(("is_last".into(), is_last_tensor.into()));
// State Tensors
let pre_conv_tensor = Tensor::from_array(state.pre_conv_history.clone().into_dyn())?;
inputs_vec.push(("pre_conv_history".into(), pre_conv_tensor.into()));
let latent_tensor = Tensor::from_array(state.latent_buffer.clone().into_dyn())?;
inputs_vec.push(("latent_buffer".into(), latent_tensor.into()));
let conv_tensor = Tensor::from_array(state.conv_history.clone().into_dyn())?;
inputs_vec.push(("conv_history".into(), conv_tensor.into()));
// KV Cache
for (i, (k, v)) in state.kv_cache.iter().enumerate() {
let k_tensor = Tensor::from_array(k.clone().into_dyn())?;
let v_tensor = Tensor::from_array(v.clone().into_dyn())?;
inputs_vec.push((format!("past_key_{}", i).into(), k_tensor.into()));
inputs_vec.push((format!("past_value_{}", i).into(), v_tensor.into()));
}
// 2. Run Session
let outputs = self.session.run(inputs_vec)?;
// 3. Extract Audio
let wav_output = &outputs["final_wav"];
let wav_raw = wav_output.try_extract_tensor::<f32>()?;
// If outputs["valid_samples"] exists, use it. Python says: outputs[1] is valid_samples.
// Let's check output names in Python: final_wav, valid_samples, pre_conv_new, ...
// We assume safe to retrieve by name.
// Handling valid_samples which might be scalar or 1D
let valid_count = if let Some(valid_out) = outputs.get("valid_samples") {
let valid_raw = valid_out.try_extract_tensor::<i64>()?;
valid_raw.1[0] as usize
} else {
wav_raw.1.len()
};
let audio: Vec<f32> = wav_raw.1.iter().take(valid_count).cloned().collect();
// 4. Update State (only if not final? Python says "if is_final, state might be dirty but unused")
// We update unconditionally to match Python logic which returns new_state.
let extract_3d = |name: &str| -> Result<Array3<f32>, Box<dyn Error>> {
let out = &outputs[name];
let raw = out.try_extract_tensor::<f32>()?;
let shape = raw.0; // &[usize] or similar
// Convert shape slices/vecs to exact dimensional array construction
// ort 2.0 API might handle shape differently. output shape is typically dynamic.
// We assume dimension correctness from model.
// ndarray::Array::from_shape_vec is simplest if we know dimensions.
// But dimensions change. We use from_iter with shape.
let shape_ix: (usize, usize, usize) =
(shape[0] as usize, shape[1] as usize, shape[2] as usize);
Ok(Array3::from_shape_vec(shape_ix, raw.1.to_vec())?)
};
state.pre_conv_history = extract_3d("next_pre_conv_history")?;
state.latent_buffer = extract_3d("next_latent_buffer")?;
state.conv_history = extract_3d("next_conv_history")?;
// Update KV Cache
// Names: next_key_0, next_value_0...
for i in 0..8 {
let k_name = format!("next_key_{}", i);
let v_name = format!("next_value_{}", i);
let out_k = &outputs[k_name.as_str()];
let raw_k = out_k.try_extract_tensor::<f32>()?;
let shape_k = (
raw_k.0[0] as usize,
raw_k.0[1] as usize,
raw_k.0[2] as usize,
raw_k.0[3] as usize,
);
let arr_k = Array4::from_shape_vec(shape_k, raw_k.1.to_vec())?;
let out_v = &outputs[v_name.as_str()];
let raw_v = out_v.try_extract_tensor::<f32>()?;
let shape_v = (
raw_v.0[0] as usize,
raw_v.0[1] as usize,
raw_v.0[2] as usize,
raw_v.0[3] as usize,
);
let arr_v = Array4::from_shape_vec(shape_v, raw_v.1.to_vec())?;
state.kv_cache[i] = (arr_k, arr_v);
}
Ok(audio)
}
}
pub struct DecoderState {
pub pre_conv_history: Array3<f32>,
pub latent_buffer: Array3<f32>,
pub conv_history: Array3<f32>,
pub kv_cache: Vec<(Array4<f32>, Array4<f32>)>,
}
impl Default for DecoderState {
fn default() -> Self {
Self::new()
}
}
impl DecoderState {
pub fn new() -> Self {
// Match Python: np.zeros((1, 512, 0)) etc.
let pre_conv_history = Array3::<f32>::zeros((1, 512, 0));
let latent_buffer = Array3::<f32>::zeros((1, 1024, 0));
let conv_history = Array3::<f32>::zeros((1, 1024, 0));
let mut kv_cache = Vec::with_capacity(8);
for _ in 0..8 {
// [1, 16, 0, 64]
let k = Array4::<f32>::zeros((1, 16, 0, 64));
let v = Array4::<f32>::zeros((1, 16, 0, 64));
kv_cache.push((k, v));
}
DecoderState {
pre_conv_history,
latent_buffer,
conv_history,
kv_cache,
}
}
}
// 兼容性保留
#[derive(Clone)]
pub struct CodecEmbeddings {
pub weights: Vec<Vec<f32>>,
pub bias: Vec<f32>,
}
impl CodecEmbeddings {
pub fn load_npz(_path: &str) -> Result<Self, Box<dyn Error>> {
Ok(CodecEmbeddings {
weights: Vec::new(),
bias: Vec::new(),
})
}
}
pub fn init_onruntime() -> Result<(), Box<dyn std::error::Error>> {
use std::path::PathBuf;
let (dll_name, providers_dll_name) = if cfg!(windows) {
("onnxruntime.dll", "onnxruntime_providers_shared.dll")
} else if cfg!(target_os = "macos") {
(
"libonnxruntime.dylib",
"libonnxruntime_providers_shared.dylib",
)
} else {
("libonnxruntime.so", "libonnxruntime_providers_shared.so")
};
let runtime_dir = PathBuf::from("runtime");
let dll_path = runtime_dir.join(dll_name);
if !dll_path.exists() {
return Err(format!("ONNX Runtime DLL not found at: {}", dll_path.display()).into());
}
let providers_path = runtime_dir.join(providers_dll_name);
if providers_path.exists() {
std::env::set_var("ORT_DYLIB_PATH", &dll_path);
}
println!(" [ONNX] Loading ONNX Runtime from: {}", dll_path.display());
let env_builder =
ort::init_from(&dll_path).map_err(|e| format!("Failed to load ONNX Runtime DLL: {}", e))?;
let committed = env_builder.with_name("Qwen3TTS").commit();
if !committed {
return Err("Failed to commit ONNX Runtime environment".into());
}
println!(" [ONNX] ONNX Runtime initialized successfully.");
Ok(())
}
+533
View File
@@ -0,0 +1,533 @@
use libloading::{Library, Symbol};
use std::error::Error;
use std::ffi::{CStr, CString};
use std::fmt;
use std::os::raw::{c_char, c_int, c_void};
use std::path::Path;
use std::ptr::null_mut;
type OrtApiBasePtr = *mut c_void;
type OrtGetApiBaseFn = unsafe extern "C" fn() -> OrtApiBasePtr;
type OrtApiPtr = *mut c_void;
type OrtGetApiFn = unsafe extern "C" fn(u32) -> OrtApiPtr;
type OrtStatusPtr = *mut c_void;
type OrtEnvPtr = *mut c_void;
type OrtSessionPtr = *mut c_void;
type OrtSessionOptionsPtr = *mut c_void;
type OrtMemoryInfoPtr = *mut c_void;
type OrtTensorPtr = *mut c_void;
type OrtValuePtr = *mut c_void;
type OrtAllocatorPtr = *mut c_void;
type OrtRunOptionsPtr = *mut c_void;
pub struct OrtApi;
pub struct OrtEnv;
pub struct OrtSession;
pub struct OrtSessionOptions;
pub struct OrtMemoryInfo;
pub struct OrtAllocator;
pub struct OrtValue;
pub struct OrtRunOptions;
#[derive(Debug)]
pub struct OnnxRuntimeError(String);
impl fmt::Display for OnnxRuntimeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "ONNX Runtime Error: {}", self.0)
}
}
impl Error for OnnxRuntimeError {}
pub struct OnnxRuntime {
lib: Library,
get_api_base: Symbol<OrtGetApiBaseFn>,
get_api: Symbol<OrtGetApiFn>,
create_env: Symbol<unsafe extern "C" fn(u32, *const c_char, *mut *mut c_void) -> OrtStatusPtr>,
release_env: Symbol<unsafe extern "C" fn(*mut c_void)>,
create_session_options: Symbol<unsafe extern "C" fn(*mut *mut c_void) -> OrtStatusPtr>,
release_session_options: Symbol<unsafe extern "C" fn(*mut c_void)>,
set_gpu_mem_limit: Symbol<unsafe extern "C" fn(*mut c_void, c_int, c_int) -> OrtStatusPtr>,
create_session: Symbol<
unsafe extern "C" fn(
*const c_char,
*mut c_void,
*mut c_void,
*mut *mut c_void,
) -> OrtStatusPtr,
>,
release_session: Symbol<unsafe extern "C" fn(*mut c_void)>,
get_input_count: Symbol<unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr>,
get_output_count: Symbol<unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr>,
get_input_name: Symbol<
unsafe extern "C" fn(*mut c_void, c_int, *mut c_void, *mut *mut c_void) -> OrtStatusPtr,
>,
get_output_name: Symbol<
unsafe extern "C" fn(*mut c_void, c_int, *mut c_void, *mut *mut c_void) -> OrtStatusPtr,
>,
release_allocator: Symbol<unsafe extern "C" fn(*mut c_void)>,
create_memory_info: Symbol<
unsafe extern "C" fn(
*mut *mut c_void,
*mut *mut c_void,
usize,
c_int,
c_int,
) -> OrtStatusPtr,
>,
release_memory_info: Symbol<unsafe extern "C" fn(*mut c_void)>,
create_tensor_with_data: Symbol<
unsafe extern "C" fn(
*mut *mut c_void,
*const c_void,
usize,
*mut c_void,
c_int,
*mut *mut c_void,
) -> OrtStatusPtr,
>,
get_tensor_type: Symbol<unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr>,
release_value: Symbol<unsafe extern "C" fn(*mut c_void)>,
run: Symbol<
unsafe extern "C" fn(
*mut c_void,
*mut c_void,
*const *const c_char,
*const *mut c_void,
c_int,
*mut *mut c_void,
) -> OrtStatusPtr,
>,
get_tensor_shape:
Symbol<unsafe extern "C" fn(*mut c_void, *mut c_int, *mut c_int) -> OrtStatusPtr>,
get_string_tensor_data:
Symbol<unsafe extern "C" fn(*mut c_void, *mut c_void, usize) -> OrtStatusPtr>,
get_string_tensor_data_length:
Symbol<unsafe extern "C" fn(*mut c_void, c_int, *mut usize) -> OrtStatusPtr>,
get_allocator_with_default_options:
Symbol<unsafe extern "C" fn(*mut *mut c_void) -> OrtStatusPtr>,
}
impl OnnxRuntime {
fn load_library() -> Result<Library, Box<dyn Error>> {
let runtime_path = std::env::current_dir()
.unwrap_or_default()
.join("runtime");
#[cfg(target_os = "windows")]
{
use std::os::windows::ffi::OsStrExt;
let path_wide = std::ffi::OsStr::new(&runtime_path)
.encode_wide()
.chain(std::iter::once(0))
.collect::<Vec<u16>>();
unsafe {
winapi::um::winbase::SetDllDirectoryW(path_wide.as_ptr());
}
let path_key = "PATH";
let path_var = std::env::var_os(path_key).unwrap_or_default();
let mut paths: Vec<_> = std::env::split_paths(&path_var).collect();
if !paths.contains(&runtime_path) {
paths.insert(0, runtime_path.clone());
let new_path = std::env::join_paths(paths).unwrap();
std::env::set_var(path_key, new_path);
}
}
#[cfg(unix)]
{
let path_key = if cfg!(target_os = "macos") {
"DYLD_LIBRARY_PATH"
} else {
"LD_LIBRARY_PATH"
};
let path_var = std::env::var_os(path_key).unwrap_or_default();
let mut paths: Vec<_> = std::env::split_paths(&path_var).collect();
if !paths.contains(&runtime_path) {
paths.insert(0, runtime_path.clone());
let new_path = std::env::join_paths(paths).unwrap();
std::env::set_var(path_key, new_path);
}
}
#[cfg(windows)]
let lib_paths = vec![
"onnxruntime.dll",
"runtime/onnxruntime.dll",
"./runtime/onnxruntime.dll",
];
#[cfg(unix)]
let lib_paths = vec![
"libonnxruntime.so",
"runtime/libonnxruntime.so",
"./runtime/libonnxruntime.so",
];
let mut last_error = None;
for path in lib_paths {
match Library::new(path) {
Ok(lib) => {
println!("Loaded ONNX Runtime from: {}", path);
return Ok(lib);
}
Err(e) => {
last_error = Some(e);
}
}
}
Err(OnnxRuntimeError(format!(
"Failed to load ONNX Runtime from any path. Last error: {}",
last_error.unwrap()
))
.into())
}
pub fn new() -> Result<Self, Box<dyn Error>> {
let lib = Self::load_library()?;
unsafe {
let get_api_base: Symbol<OrtGetApiBaseFn> = lib.get(b"OrtGetApiBase")?;
let get_api: Symbol<OrtGetApiFn> = lib.get(b"OrtGetApi")?;
let api_base = get_api_base();
if api_base.is_null() {
return Err(OnnxRuntimeError("Failed to get ONNX API base".into()).into());
}
let api = get_api(16);
if api.is_null() {
return Err(OnnxRuntimeError("Failed to get ONNX API v16".into()).into());
}
let create_env: Symbol<
unsafe extern "C" fn(u32, *const c_char, *mut *mut c_void) -> OrtStatusPtr,
> = lib.get(b"OrtCreateEnv")?;
let release_env: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseEnv")?;
let create_session_options: Symbol<
unsafe extern "C" fn(*mut *mut c_void) -> OrtStatusPtr,
> = lib.get(b"OrtCreateSessionOptions")?;
let release_session_options: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseSessionOptions")?;
let set_gpu_mem_limit: Symbol<
unsafe extern "C" fn(*mut c_void, c_int, c_int) -> OrtStatusPtr,
> = lib.get(b"OrtSetSessionMemoryAlignment")?;
let create_session: Symbol<
unsafe extern "C" fn(
*const c_char,
*mut c_void,
*mut c_void,
*mut *mut c_void,
) -> OrtStatusPtr,
> = lib.get(b"OrtCreateSession")?;
let release_session: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseSession")?;
let get_input_count: Symbol<
unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr,
> = lib.get(b"OrtSessionGetInputCount")?;
let get_output_count: Symbol<
unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr,
> = lib.get(b"OrtSessionGetOutputCount")?;
let get_input_name: Symbol<
unsafe extern "C" fn(
*mut c_void,
c_int,
*mut c_void,
*mut *mut c_void,
) -> OrtStatusPtr,
> = lib.get(b"OrtSessionGetInputName")?;
let get_output_name: Symbol<
unsafe extern "C" fn(
*mut c_void,
c_int,
*mut c_void,
*mut *mut c_void,
) -> OrtStatusPtr,
> = lib.get(b"OrtSessionGetOutputName")?;
let release_allocator: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseAllocator")?;
let create_memory_info: Symbol<
unsafe extern "C" fn(
*mut *mut c_void,
*mut *mut c_void,
usize,
c_int,
c_int,
) -> OrtStatusPtr,
> = lib.get(b"OrtCreateMemoryInfo")?;
let release_memory_info: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseMemoryInfo")?;
let create_tensor_with_data: Symbol<
unsafe extern "C" fn(
*mut *mut c_void,
*const c_void,
usize,
*mut c_void,
c_int,
*mut *mut c_void,
) -> OrtStatusPtr,
> = lib.get(b"OrtCreateTensorWithDataAsOrtValue")?;
let get_tensor_type: Symbol<
unsafe extern "C" fn(*mut c_void, *mut c_int) -> OrtStatusPtr,
> = lib.get(b"OrtGetTensorType")?;
let release_value: Symbol<unsafe extern "C" fn(*mut c_void)> =
lib.get(b"OrtReleaseValue")?;
let run: Symbol<
unsafe extern "C" fn(
*mut c_void,
*mut c_void,
*const *const c_char,
*const *mut c_void,
c_int,
*mut *mut c_void,
) -> OrtStatusPtr,
> = lib.get(b"OrtRun")?;
let get_tensor_shape: Symbol<
unsafe extern "C" fn(*mut c_void, *mut c_int, *mut c_int) -> OrtStatusPtr,
> = lib.get(b"OrtGetTensorShape")?;
let get_allocator_with_default_options: Symbol<
unsafe extern "C" fn(*mut *mut c_void) -> OrtStatusPtr,
> = lib.get(b"OrtGetAllocatorWithDefaultOptions")?;
Ok(OnnxRuntime {
lib,
get_api_base,
get_api,
create_env,
release_env,
create_session_options,
release_session_options,
set_gpu_mem_limit,
create_session,
release_session,
get_input_count,
get_output_count,
get_input_name,
get_output_name,
release_allocator,
create_memory_info,
release_memory_info,
create_tensor_with_data,
get_tensor_type,
release_value,
run,
get_tensor_shape,
get_string_tensor_data,
get_string_tensor_data_length,
get_allocator_with_default_options,
})
}
}
}
pub struct OnnxSession {
runtime: *mut OnnxRuntime,
session: OrtSessionPtr,
input_names: Vec<String>,
output_names: Vec<String>,
}
impl OnnxSession {
pub fn load(runtime: &OnnxRuntime, model_path: &Path) -> Result<Self, Box<dyn Error>> {
let mut session: OrtSessionPtr = null_mut();
unsafe {
let model_path_cstr = std::ffi::CString::new(model_path.to_string_lossy().as_bytes())?;
let status = (runtime.create_session)(
model_path_cstr.as_ptr(),
runtime as *const _ as *mut c_void,
null_mut(),
&mut session,
);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to create ONNX session".into()).into());
}
let mut input_count: c_int = 0;
let mut output_count: c_int = 0;
let status = (runtime.get_input_count)(session, &mut input_count);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to get input count".into()).into());
}
let status = (runtime.get_output_count)(session, &mut output_count);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to get output count".into()).into());
}
let mut allocator: OrtAllocatorPtr = null_mut();
let status = (runtime.get_allocator_with_default_options)(&mut allocator);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to get default allocator".into()).into());
}
let mut input_names = Vec::new();
for i in 0..input_count {
let mut name_ptr: *mut c_char = null_mut();
let status = (runtime.get_input_name)(
session,
i,
allocator,
&mut name_ptr as *mut *mut c_char as *mut *mut c_void,
);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to get input name".into()).into());
}
let name = CStr::from_ptr(name_ptr).to_string_lossy().into_owned();
input_names.push(name);
// Note: We are leaking the name string memory here because we don't have OrtAllocatorFree bound
}
let mut output_names = Vec::new();
for i in 0..output_count {
let mut name_ptr: *mut c_char = null_mut();
let status = (runtime.get_output_name)(
session,
i,
allocator,
&mut name_ptr as *mut *mut c_char as *mut *mut c_void,
);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to get output name".into()).into());
}
let name = CStr::from_ptr(name_ptr).to_string_lossy().into_owned();
output_names.push(name);
}
Ok(OnnxSession {
runtime: runtime as *const _ as *mut OnnxRuntime,
session,
input_names,
output_names,
})
}
}
pub fn run(
&self,
runtime: &OnnxRuntime,
inputs: &[&[f32]],
output_shapes: &[Vec<i64>],
) -> Result<Vec<Vec<f32>>, Box<dyn Error>> {
let mut output_values: Vec<OrtValuePtr> = Vec::new();
let mut input_values: Vec<OrtValuePtr> = Vec::new();
let mut allocator: OrtAllocatorPtr = null_mut();
let mut memory_info: OrtMemoryInfoPtr = null_mut();
unsafe {
let status = (runtime.create_memory_info)(&mut memory_info, &mut allocator, 0, 1, 0);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to create memory info".into()).into());
}
for (i, input_data) in inputs.iter().enumerate() {
let shape: [i64; 2] = [1, input_data.len() as i64];
let mut value: OrtValuePtr = null_mut();
let status = (runtime.create_tensor_with_data)(
&mut value,
input_data.as_ptr() as *const c_void,
(input_data.len() * std::mem::size_of::<f32>()) as usize,
memory_info,
2,
shape.as_ptr() as *mut c_void,
);
if !status.is_null() {
return Err(OnnxRuntimeError("Failed to create input tensor".into()).into());
}
input_values.push(value);
}
let input_names_cstring: Result<Vec<CString>, _> = self
.input_names
.iter()
.map(|s| CString::new(s.as_str()))
.collect();
let input_names_cstring = input_names_cstring?;
let input_names_ptr: Vec<*const c_char> =
input_names_cstring.iter().map(|cs| cs.as_ptr()).collect();
let mut run_options: OrtRunOptionsPtr = null_mut();
let status = (runtime.run)(
self.session,
run_options,
input_names_ptr.as_ptr(),
input_values.as_ptr() as *const *mut OrtValuePtr,
input_names_ptr.len() as c_int,
output_values.as_mut_ptr(),
output_shapes.len() as i32,
);
for value in input_values {
(runtime.release_value)(value);
}
(runtime.release_memory_info)(memory_info);
if !status.is_null() {
return Err(OnnxRuntimeError("ONNX run failed".into()).into());
}
output_values.set_len(output_shapes.len());
let mut outputs = Vec::new();
for (i, shape) in output_shapes.iter().enumerate() {
let value = output_values[i];
let mut tensor_type: c_int = 0;
let status = (runtime.get_tensor_type)(value, &mut tensor_type);
if !status.is_null() {
for v in &output_values {
(runtime.release_value)(*v);
}
return Err(OnnxRuntimeError("Failed to get tensor type".into()).into());
}
let mut dims: [c_int; 4] = [0, 0, 0, 0];
let status = (runtime.get_tensor_shape)(value, dims.as_mut_ptr(), 4);
if !status.is_null() {
for v in &output_values {
(runtime.release_value)(*v);
}
return Err(OnnxRuntimeError("Failed to get tensor shape".into()).into());
}
let total_size: usize = shape.iter().product::<i64>() as usize;
let mut output_data: Vec<f32> = vec![0.0; total_size];
let data_ptr = output_data.as_mut_ptr() as *mut c_void;
std::ptr::copy_nonoverlapping(
value as *const c_void,
data_ptr,
total_size * std::mem::size_of::<f32>(),
);
outputs.push(output_data);
}
for value in output_values {
(runtime.release_value)(value);
}
Ok(outputs)
}
}
}
impl Drop for OnnxSession {
fn drop(&mut self) {
unsafe {
if !self.session.is_null() {
let runtime = &*self.runtime;
(runtime.release_session)(self.session);
}
}
}
}
+849
View File
@@ -0,0 +1,849 @@
use crate::assets_manager::Assets;
use crate::models::llama::{LlamaBatch, LlamaContext, LlamaModel, LlamaSampler};
use crate::models::onnx::{AudioDecoder, AudioEncoder, SpeakerEncoder};
use crate::tts::prompt::PromptBuilder;
use crate::utils::cache;
use crate::utils::tokenizer::Tokenizer;
use crate::utils::voice_file::VoiceFile;
use crate::AudioSample;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
/// Sampler configuration for TTS generation
#[derive(Debug, Clone)]
pub struct SamplerConfig {
/// Temperature for sampling (higher = more random, 0.0 = greedy)
pub temperature: f32,
/// Top-K sampling (0 = disabled)
pub top_k: i32,
/// Top-P (nucleus) sampling (1.0 = disabled)
pub top_p: f32,
/// Min-P sampling threshold (0.0 = disabled)
pub min_p: f32,
/// Repeat penalty (1.0 = disabled)
pub repeat_penalty: f32,
/// Frequency penalty (0.0 = disabled)
pub frequency_penalty: f32,
/// Presence penalty (0.0 = disabled)
pub presence_penalty: f32,
/// Number of recent tokens to consider for penalties
pub penalty_last_n: usize,
/// Random seed (None = use system entropy)
pub seed: Option<u64>,
}
impl Default for SamplerConfig {
fn default() -> Self {
Self {
temperature: 0.9,
top_k: 50,
top_p: 1.0,
min_p: 0.0,
repeat_penalty: 1.05,
frequency_penalty: 0.0,
presence_penalty: 0.0,
penalty_last_n: 128,
seed: None,
}
}
}
impl SamplerConfig {
pub fn new(temperature: f32, top_k: i32, top_p: f32, seed: Option<u64>) -> Self {
Self {
temperature,
top_k,
top_p,
min_p: 0.0,
repeat_penalty: 1.05,
frequency_penalty: 0.0,
presence_penalty: 0.0,
penalty_last_n: 128,
seed,
}
}
pub fn with_penalties(
mut self,
min_p: f32,
repeat_penalty: f32,
frequency_penalty: f32,
presence_penalty: f32,
penalty_last_n: usize,
) -> Self {
self.min_p = min_p;
self.repeat_penalty = repeat_penalty;
self.frequency_penalty = frequency_penalty;
self.presence_penalty = presence_penalty;
self.penalty_last_n = penalty_last_n;
self
}
}
/// Main TTS Engine Struct
///
/// IMPORTANT: Field ordering matters for Drop!
/// Rust drops fields in declaration order. Contexts MUST be declared before models
/// because context destructors reference model memory. If models are dropped first,
/// context destructors will access freed memory (ACCESS_VIOLATION).
pub struct TtsEngine {
assets: Assets,
tokenizer: Tokenizer,
// ONNX Models
encoder: Option<AudioEncoder>,
speaker_encoder: Option<SpeakerEncoder>,
decoder: Arc<Mutex<Option<AudioDecoder>>>,
// Llama: Contexts MUST be listed before models for correct drop order
talker_ctx: LlamaContext,
predictor_ctx: LlamaContext,
talker_model: LlamaModel,
predictor_model: LlamaModel,
// Speakers Cache
speakers: HashMap<String, VoiceFile>,
// Config
_model_dir: PathBuf,
max_steps: usize,
sampler_config: SamplerConfig,
}
impl TtsEngine {
/// Initialize the TTS Engine from the specified model directory.
///
/// This function loads all necessary models (GGUF, Onnx, Tokenizer) from the given directory.
/// It ensures that the essential components for inference are present.
///
/// # Arguments
///
/// * `model_dir` - Path to the directory containing model files.
/// * `quant` - Quantization level (e.g., "none", "q5_k_m", "q8_0").
/// * `n_threads` - Number of threads to use for generation (default: 4 if <= 0).
pub async fn new(
model_dir: impl AsRef<Path>,
quant: &str,
_n_threads: i32,
) -> Result<Self, String> {
let model_dir = model_dir.as_ref();
println!("Loading TtsEngine from: {:?} (quant: {})", model_dir, quant);
// 0. Auto-download check (Models + Runtimes)
Self::download_models(model_dir, quant).await?;
let quant_dir = match quant {
"q5_k_m" => "gguf_q5_k_m",
"q8_0" => "gguf_q8_0",
_ => "gguf",
};
// 1. Assets - 总是从未量化的 gguf 目录加载(assets 不应该被量化)
let assets_path = model_dir.join("gguf");
let assets =
Assets::load(&assets_path).map_err(|e| format!("Failed to load assets: {}", e))?;
// 2. Tokenizer
let tokenizer =
Tokenizer::load(model_dir).map_err(|e| format!("Failed to load tokenizer: {}", e))?;
// 3. ONNX Runtime is initialized lazily by the first Session builder.
// The runtime DLL still needs to be discoverable at load time, but we avoid
// an eager global environment commit here because it can hang on some Linux setups.
println!("Initializing ONNX Runtime (lazy)...");
// 4. ONNX Models (Optional for preset mode, but good to have)
let onnx_dir = model_dir.join("onnx");
let encoder = AudioEncoder::load(
&onnx_dir
.join("qwen3_tts_codec_encoder.onnx")
.to_string_lossy(),
)
.ok();
let speaker_encoder = SpeakerEncoder::load(
&onnx_dir
.join("qwen3_tts_speaker_encoder.onnx")
.to_string_lossy(),
)
.ok();
// 5. Load GGUF Models
let talker_path = model_dir.join(quant_dir).join("qwen3_tts_talker.gguf");
let predictor_path = model_dir.join(quant_dir).join("qwen3_tts_predictor.gguf");
let talker_model = LlamaModel::load(&talker_path, 99)
.map_err(|e| format!("Failed to load Talker: {}", e))?;
let predictor_model = LlamaModel::load(&predictor_path, 99)
.map_err(|e| format!("Failed to load Predictor: {}", e))?;
// 5. Create Contexts
// talker: n_ctx=4096, n_batch=2048, embeddings=1, threads=-1 (auto)
let talker_ctx = LlamaContext::new(&talker_model, 4096, 2048, 1, -1)
.map_err(|e| format!("Failed to create Talker context: {}", e))?;
// predictor: n_ctx=512, n_batch=32, embeddings=0, threads=4
let predictor_ctx = LlamaContext::new(&predictor_model, 512, 32, 0, 4)
.map_err(|e| format!("Failed to create Predictor context: {}", e))?;
// 6. 异步预加载 decoder(后台加载以避免阻塞启动)。如果后台加载失败
//,会在实际需要解码时尝试延迟加载。
println!("Starting background preload for AudioDecoder...");
let decoder: Arc<Mutex<Option<AudioDecoder>>> = Arc::new(Mutex::new(None));
{
let decoder_clone = decoder.clone();
let decoder_path = onnx_dir
.join("qwen3_tts_decoder.onnx")
.to_string_lossy()
.to_string();
std::thread::spawn(move || match AudioDecoder::load(&decoder_path) {
Ok(d) => {
let mut guard = decoder_clone.lock().unwrap();
*guard = Some(d);
println!("AudioDecoder loaded in background.");
}
Err(e) => {
println!("Warning: failed to preload AudioDecoder: {}", e);
}
});
}
println!("AudioDecoder background preload started.");
println!("TtsEngine loaded successfully.");
let mut engine = Self {
assets,
tokenizer,
encoder,
speaker_encoder,
decoder,
talker_model,
predictor_model,
talker_ctx,
predictor_ctx,
speakers: HashMap::new(),
_model_dir: model_dir.to_path_buf(),
max_steps: 4096,
sampler_config: SamplerConfig::default(),
};
// 6. Load Speakers
let speakers_dir = model_dir.join("preset_speakers"); // Default to preset directory
let speakers_dir = if speakers_dir.exists() {
speakers_dir
} else {
PathBuf::from("speakers")
};
if speakers_dir.exists() {
engine.load_speakers(&speakers_dir)?;
}
Ok(engine)
}
/// Set the maximum number of generation steps (tokens).
pub fn set_max_steps(&mut self, steps: usize) {
self.max_steps = steps;
}
/// Set the sampler configuration for generation.
pub fn set_sampler_config(&mut self, config: SamplerConfig) {
self.sampler_config = config;
}
/// Get the current sampler configuration.
pub fn get_sampler_config(&self) -> &SamplerConfig {
&self.sampler_config
}
/// Get mutable sampler configuration.
pub fn get_sampler_config_mut(&mut self) -> &mut SamplerConfig {
&mut self.sampler_config
}
/// Get the speakers map.
pub fn get_speakers_map(&self) -> &HashMap<String, VoiceFile> {
&self.speakers
}
/// Load all speakers from the specified directory.
pub fn load_speakers(&mut self, speakers_dir: impl AsRef<Path>) -> Result<(), String> {
let speakers_dir = speakers_dir.as_ref();
println!("Loading speakers from: {:?}", speakers_dir);
let entries = std::fs::read_dir(speakers_dir).map_err(|e| e.to_string())?;
for entry in entries {
let entry = entry.map_err(|e| e.to_string())?;
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) == Some("json") {
if let Ok(voice) = VoiceFile::load(&path) {
let id = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("unknown")
.to_string();
self.speakers.insert(id, voice);
}
}
}
println!("Loaded {} speakers.", self.speakers.len());
Ok(())
}
/// Get a speaker by ID or name, with fallback to "vivian".
pub fn get_speaker(&self, id_or_name: &str) -> &VoiceFile {
if let Some(v) = self.speakers.get(id_or_name) {
return v;
}
// Fallback to name match
for v in self.speakers.values() {
if let Some(ref name) = v.name {
if name == id_or_name {
return v;
}
}
}
// Final fallback to vivian
self.speakers.get("vivian").unwrap_or_else(|| {
// Panic if even vivian is missing and cache is empty
self.speakers
.values()
.next()
.expect("No speakers loaded in engine!")
})
}
/// Helper to download necessary files before loading.
pub async fn download_models(model_dir: impl AsRef<Path>, quant: &str) -> Result<(), String> {
let downloader = crate::download::Downloader::new().await;
downloader
.check_and_download(model_dir.as_ref(), quant)
.await
.map_err(|e| format!("Download failed: {}", e))
}
/// Generate speech from text using a reference audio.
pub fn generate(
&mut self,
text: &str,
ref_audio_path: impl AsRef<Path>,
ref_text: &str,
instruct: Option<&str>,
) -> Result<AudioSample, String> {
let ref_audio_path = ref_audio_path.as_ref();
// 1. Process Reference Audio
let (ref_codes, spk_emb) = self.process_reference(ref_audio_path)?;
// 2. Build Prompt
// lang_id = 2055 (Chinese) hardcoded for now or parameterize later
let ref_text_ids = self.tokenizer.encode(ref_text);
let ref_codes_i32: Vec<i32> = ref_codes.iter().map(|&c| c as i32).collect();
let prompt_data = PromptBuilder::build_clone_prompt(
text,
&self.tokenizer,
&self.assets,
&ref_codes_i32,
&ref_text_ids,
&spk_emb,
2055,
instruct,
);
self.run_inference(prompt_data)
}
/// Process reference audio to get codes and speaker embedding, using cache if available.
fn process_reference(&mut self, audio_path: &Path) -> Result<(Vec<i64>, Vec<f32>), String> {
let cache_path = audio_path.with_extension("cache");
if cache_path.exists() {
if let Ok((c, e)) = cache::load_cache(&cache_path) {
return Ok((c, e));
}
}
let audio = AudioSample::load_wav(audio_path)
.map_err(|e| format!("Failed to load audio: {}", e))?;
let ref_codes = self
.encoder
.as_mut()
.ok_or("AudioEncoder not loaded (required for processing raw audio)".to_string())?
.encode(&audio.samples)
.map_err(|e| format!("Audio encode failed: {}", e))?;
let spk_emb = self
.speaker_encoder
.as_mut()
.ok_or("SpeakerEncoder not loaded (required for processing raw audio)".to_string())?
.encode(&audio.samples)
.map_err(|e| format!("Speaker extraction failed: {}", e))?;
let _ = cache::save_cache(&cache_path, &ref_codes, &spk_emb);
Ok((ref_codes, spk_emb))
}
// --- Helpers ---
fn qwen3_position(start: i32, len: usize) -> Vec<i32> {
let mut pos = Vec::with_capacity(len * 4);
let range: Vec<i32> = (start..start + len as i32).collect();
pos.extend_from_slice(&range); // Temporal
pos.extend_from_slice(&range); // Height
pos.extend_from_slice(&range); // Width
pos.extend(std::iter::repeat_n(0, len)); // Channel
pos
}
fn normal_position(cur_pos: usize, n_tokens: usize) -> Vec<i32> {
(0..n_tokens).map(|i| (cur_pos + i) as i32).collect()
}
/// Create a VoiceFile from a reference audio file and its text.
///
/// Requires that AudioEncoder and SpeakerEncoder are loaded.
/// The reference audio MUST be 24000Hz.
pub fn create_voice_file(
&mut self,
audio_path: impl AsRef<Path>,
ref_text: String,
) -> Result<crate::utils::voice_file::VoiceFile, String> {
let encoder = self.encoder.as_mut().ok_or(
"AudioEncoder not loaded. Please ensure models/onnx/qwen3_tts_codec_encoder.onnx exists.",
)?;
let speaker_encoder = self.speaker_encoder.as_mut().ok_or(
"SpeakerEncoder not loaded. Please ensure models/onnx/qwen3_tts_speaker_encoder.onnx exists.",
)?;
// 1. Load Audio
let mut reader =
hound::WavReader::open(audio_path).map_err(|e| format!("WAV error: {}", e))?;
let spec = reader.spec();
if spec.sample_rate != 24000 {
return Err(format!(
"Expected 24000Hz audio, found {}Hz",
spec.sample_rate
));
}
let audio: Vec<f32> = match (spec.sample_format, spec.bits_per_sample) {
(hound::SampleFormat::Float, 32) => {
reader.samples::<f32>().map(|s| s.unwrap_or(0.0)).collect()
}
(hound::SampleFormat::Int, 16) => reader
.samples::<i16>()
.map(|s| (s.unwrap_or(0) as f32) / 32768.0)
.collect(),
(hound::SampleFormat::Int, 32) => reader
.samples::<i32>()
.map(|s| (s.unwrap_or(0) as f32) / 2147483648.0)
.collect(),
_ => {
return Err(format!(
"Unsupported WAV format: {:?} {} bits",
spec.sample_format, spec.bits_per_sample
))
}
};
// If stereo, take channel 1
let audio = if spec.channels > 1 {
audio.chunks(spec.channels as usize).map(|c| c[0]).collect()
} else {
audio
};
// 2. Run Encoders
println!("Extracting audio codes...");
let audio_codes = encoder.encode(&audio).map_err(|e| e.to_string())?;
println!("Extracting speaker embedding...");
let speaker_embedding = speaker_encoder.encode(&audio).map_err(|e| e.to_string())?;
Ok(crate::utils::voice_file::VoiceFile::new(
ref_text,
audio_codes,
speaker_embedding,
))
}
/// Generate speech using a pre-loaded VoiceFile.
pub fn generate_with_voice(
&mut self,
text: &str,
voice: &crate::VoiceFile,
instruct: Option<&str>,
) -> Result<AudioSample, String> {
self.generate_with_voice_streaming(text, voice, instruct, None)
}
/// Generate speech using a pre-loaded VoiceFile.
pub fn generate_with_voice_streaming(
&mut self,
text: &str,
voice: &crate::VoiceFile,
instruct: Option<&str>,
stream_tx: Option<tokio::sync::mpsc::UnboundedSender<Vec<f32>>>,
) -> Result<AudioSample, String> {
let prompt_data = if voice.audio_codes.is_empty() {
PromptBuilder::build_core(
text,
&self.tokenizer,
&self.assets,
Some(2055),
None,
Some(&voice.speaker_embedding),
instruct,
None,
)
} else {
let ref_text_ids = self.tokenizer.encode(&voice.ref_text);
let ref_codes_i32: Vec<i32> = voice.audio_codes.iter().map(|&c| c as i32).collect();
Ok(PromptBuilder::build_clone_prompt(
text,
&self.tokenizer,
&self.assets,
&ref_codes_i32,
&ref_text_ids,
&voice.speaker_embedding,
2055,
instruct,
))
}?;
self.run_inference_stream(prompt_data, stream_tx)
}
fn run_inference(
&mut self,
prompt_data: crate::tts::prompt::PromptData,
) -> Result<AudioSample, String> {
self.run_inference_stream(prompt_data, None)
}
fn run_inference_stream(
&mut self,
prompt_data: crate::tts::prompt::PromptData,
stream_tx: Option<tokio::sync::mpsc::UnboundedSender<Vec<f32>>>,
) -> Result<AudioSample, String> {
self.talker_ctx.clear_kv_cache();
self.predictor_ctx.clear_kv_cache();
let n_tokens_prompt = prompt_data.embd.len();
let prompt_embeds_flat: Vec<f32> = prompt_data.embd.iter().flatten().copied().collect();
let talker_embd = self.talker_model.n_embd;
let predictor_embd = self.predictor_model.n_embd;
let mut talker_batch = LlamaBatch::new(4096, talker_embd, 1, 4);
let pos_arr = Self::qwen3_position(0, n_tokens_prompt);
talker_batch.set_embd(&prompt_embeds_flat, &pos_arr, 0);
self.talker_ctx
.decode(&mut talker_batch)
.map_err(|e| format!("Talker prefill failed: {}", e))?;
let mut all_codes: Vec<i32> = Vec::new();
let mut cur_pos = n_tokens_prompt;
let mut predictor_batch = LlamaBatch::new(32, predictor_embd, 1, 1);
let predictor_sampler = LlamaSampler::greedy(self.predictor_model.n_vocab);
let seed = self.sampler_config.seed.unwrap_or_else(|| {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or_else(|_| rand::random())
});
let talker_sampler = LlamaSampler::new(
self.talker_model.n_vocab,
self.sampler_config.temperature,
self.sampler_config.top_k,
self.sampler_config.top_p,
seed,
);
let (tx, rx) = std::sync::mpsc::channel::<(Vec<i64>, bool)>();
let tts_pad = self.assets.tts_pad.clone();
let decoder_arc = self.decoder.clone();
let decoder_model_path = self
._model_dir
.join("onnx")
.join("qwen3_tts_decoder.onnx")
.to_string_lossy()
.to_string();
let decoder_handle = std::thread::spawn(move || {
let mut full_audio = Vec::new();
// Ensure decoder is available: if background preload didn't set it,
// attempt to load lazily here. If loading fails, abort decoding.
{
let mut guard = decoder_arc.lock().unwrap();
if guard.is_none() {
match AudioDecoder::load(&decoder_model_path) {
Ok(d) => {
*guard = Some(d);
println!("AudioDecoder loaded lazily in decoder thread.");
}
Err(e) => {
eprintln!("Failed to load decoder in thread: {}", e);
return Vec::new();
}
}
}
}
let mut state = AudioDecoder::create_state();
while let Ok((codes, is_final)) = rx.recv() {
let n_frames = codes.len() / 16;
if n_frames == 0 {
if is_final {
let mut guard = decoder_arc.lock().unwrap();
if let Some(ref mut local_decoder) = guard.as_mut() {
if let Ok(samples) = local_decoder.decode(&[], &mut state, true) {
if !samples.is_empty() {
if let Some(ref stx) = stream_tx {
let _ = stx.send(samples.clone());
}
full_audio.extend(samples);
}
}
}
break;
}
continue;
}
let safe_codes: Vec<i64> = codes.iter().map(|&c| c.clamp(0, 2047)).collect();
let mut guard = decoder_arc.lock().unwrap();
if let Some(ref mut local_decoder) = guard.as_mut() {
if let Ok(samples) = local_decoder.decode(&safe_codes, &mut state, is_final) {
if !samples.is_empty() {
if let Some(ref stx) = stream_tx {
let _ = stx.send(samples.clone());
}
full_audio.extend(samples);
}
}
}
}
full_audio
});
// 连续静音检测参数 (12.5Hz = 80ms/帧)
const MAX_SILENT_FRAMES: usize = 15; // 1.2秒强制停止
const SILENT_PENALTY_THRESHOLD: usize = 4; // 0.3秒(4帧)后开始惩罚
const SILENT_PENALTY_MAX_FRAMES: usize = 8; // 0.6秒(8帧)后极大惩罚
const SILENT_PENALTY_VALUE: f32 = 2.0;
// 流式发送参数
const SEND_INTERVAL: usize = 4;
let mut consecutive_silent_frames: usize = 0;
let mut last_sent_frame: usize = 0;
for step in 0..self.max_steps {
print!("\r Generation Step {}/{}...", step + 1, self.max_steps);
let _ = std::io::Write::flush(&mut std::io::stdout());
// Talker
let sample_idx = if cur_pos == n_tokens_prompt {
(n_tokens_prompt - 1) as i32
} else {
0
};
let allow_tokens: Vec<i32> = vec![2150, 2148, 2149];
let code_0 = if consecutive_silent_frames >= SILENT_PENALTY_THRESHOLD {
let penalty = if consecutive_silent_frames >= SILENT_PENALTY_MAX_FRAMES {
100.0 // 极大惩罚,强制退出静音
} else {
SILENT_PENALTY_VALUE
* (1.0
+ (consecutive_silent_frames - SILENT_PENALTY_THRESHOLD) as f32 * 0.5)
};
eprintln!(
"[Debug] silent {} frames, penalty: {:.1}",
consecutive_silent_frames, penalty
);
talker_sampler.sample_with_silent_penalty(
&self.talker_ctx,
sample_idx,
Some(0),
Some(2048),
Some(&allow_tokens),
penalty,
SILENT_PENALTY_MAX_FRAMES, // 只对前8帧应用惩罚
)
} else {
talker_sampler.sample_with_allow(
&self.talker_ctx,
sample_idx,
Some(0),
Some(2048),
Some(&allow_tokens),
)
};
eprintln!("[Debug] Sampled code_0={}", code_0);
if code_0 == 2150 || code_0 == 151673 {
println!("\n EOS detected at step {} (code_0={})", step, code_0);
break;
}
let is_silent_frame = code_0 < 100;
if is_silent_frame {
consecutive_silent_frames += 1;
eprintln!(
"[Debug] silent frame {}, code_0={}",
consecutive_silent_frames, code_0
);
if consecutive_silent_frames >= MAX_SILENT_FRAMES {
eprintln!(
"[Debug] Early stop: {} consecutive silent frames",
consecutive_silent_frames
);
println!(
"\n Early stop: {} consecutive silent frames detected",
consecutive_silent_frames
);
break;
}
} else {
consecutive_silent_frames = 0;
}
all_codes.push(code_0);
// Predictor
let emb_idx = if step == 0 { n_tokens_prompt - 1 } else { 0 };
let m_hidden = self.talker_ctx.get_embedding_at(emb_idx).to_vec();
let m_h_1024 = self.assets.project(&m_hidden);
let code_0_1024 = self.assets.get_codec_embedding_1024(0, code_0);
let mut predictor_input = Vec::with_capacity(2 * predictor_embd);
predictor_input.extend_from_slice(&m_h_1024);
predictor_input.extend_from_slice(&code_0_1024);
self.predictor_ctx.clear_kv_cache();
predictor_batch.clear();
let pred_pos = Self::normal_position(0, 2);
predictor_batch.set_embd(&predictor_input, &pred_pos, 0);
self.predictor_ctx
.decode(&mut predictor_batch)
.map_err(|e| format!("Predictor prefill failed: {}", e))?;
let mut step_embeds_2048: Vec<Vec<f32>> = Vec::new();
step_embeds_2048.push(self.assets.get_codec_embedding(0, code_0));
for q in 1..16 {
let start_offset = (q - 1) * 2048;
let end_offset = q * 2048;
let sampled = predictor_sampler.sample(
&self.predictor_ctx,
0,
Some(start_offset),
Some(end_offset),
);
let code_q = sampled - start_offset as i32;
all_codes.push(code_q);
let emb = self.assets.get_codec_embedding(q, code_q);
step_embeds_2048.push(emb.to_vec());
if q < 15 {
let next_embed_1024 = self.assets.get_codec_embedding_1024(q, code_q);
let next_pos = Self::normal_position(q + 1, 1);
predictor_batch.clear();
predictor_batch.set_embd(&next_embed_1024, &next_pos, 0);
self.predictor_ctx
.decode(&mut predictor_batch)
.map_err(|e| format!("Predictor decode failed: {}", e))?;
}
}
// 流式发送
let current_frame = all_codes.len() / 16;
if current_frame > last_sent_frame + SEND_INTERVAL {
let start_frame = last_sent_frame;
let end_frame = last_sent_frame + SEND_INTERVAL;
let start_idx = start_frame * 16;
let end_idx = end_frame * 16;
let frame_codes: Vec<i64> = all_codes[start_idx..end_idx]
.iter()
.map(|&c| c as i64)
.collect();
let _ = tx.send((frame_codes, false));
last_sent_frame += SEND_INTERVAL;
}
let mut feedback = vec![0.0f32; 2048];
for embed in &step_embeds_2048 {
for (i, val) in embed.iter().enumerate() {
feedback[i] += val;
}
}
let text_vec = &tts_pad;
for (i, val) in text_vec.iter().enumerate() {
if i < feedback.len() {
feedback[i] += val;
}
}
feedback.resize(talker_embd, 0.0);
let talker_pos = Self::qwen3_position(cur_pos as i32, 1);
talker_batch.clear();
talker_batch.set_embd(&feedback, &talker_pos, 0);
self.talker_ctx
.decode(&mut talker_batch)
.map_err(|e| format!("Talker step failed: {}", e))?;
cur_pos += 1;
}
// 发送剩余的帧
let total_frames = all_codes.len() / 16;
if total_frames > last_sent_frame {
let start_frame = last_sent_frame;
let end_frame = total_frames;
let start_idx = start_frame * 16;
let end_idx = end_frame * 16;
let frame_codes: Vec<i64> = all_codes[start_idx..end_idx]
.iter()
.map(|&c| c as i64)
.collect();
let _ = tx.send((frame_codes, true));
} else {
let _ = tx.send((Vec::new(), true));
}
drop(tx);
let audio_samples = decoder_handle
.join()
.map_err(|_| "Decoder thread panicked".to_string())?;
Ok(AudioSample {
samples: audio_samples,
sample_rate: 24000,
channels: 1,
})
}
}
+2
View File
@@ -0,0 +1,2 @@
pub mod engine;
pub mod prompt;
+394
View File
@@ -0,0 +1,394 @@
use crate::assets_manager::Assets;
use crate::utils::tokenizer::Tokenizer;
// Constants from PROTOCOL
pub const PAD: usize = 2148;
pub const BOS: usize = 2149;
pub const EOS: usize = 2150;
pub const BOS_TOKEN: usize = 151672;
pub const EOS_TOKEN: usize = 151673;
pub const THINK: usize = 2154;
pub const NOTHINK: usize = 2155;
pub const THINK_BOS: usize = 2156;
pub const THINK_EOS: usize = 2157;
// Magic token derived from Python implementation (assets.text_table[151671])
pub const TEXT_AUDIO_MARKER: usize = 151671;
pub struct PromptData {
pub embd: Vec<Vec<f32>>,
pub text_ids: Vec<u32>,
pub spk_emb: Vec<f32>,
}
pub struct PromptBuilder;
impl PromptBuilder {
#[allow(clippy::too_many_arguments)]
pub fn build_clone_prompt(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
ref_codes: &[i32],
ref_text_ids: &[u32],
spk_emb: &[f32],
lang_id: usize,
instruct: Option<&str>,
) -> PromptData {
let tts_pad = &assets.tts_pad;
let text_ids = tokenizer.encode(text);
let mut text_pool_ids: Vec<u32> = ref_text_ids.to_vec();
text_pool_ids.extend(text_ids.iter().copied());
text_pool_ids.push(EOS_TOKEN as u32);
let text_pool: Vec<Vec<f32>> = text_pool_ids
.iter()
.map(|&id| assets.get_text_embedding(id as usize))
.collect();
let mut audio_pool: Vec<Vec<f32>> = Vec::new();
audio_pool.push(assets.get_codec_embedding(0, BOS as i32));
let n_steps = ref_codes.len() / 16;
for step in 0..n_steps {
let mut step_sum = vec![0.0f32; 2048];
for q in 0..16 {
let c = ref_codes[step * 16 + q];
let emb = assets.get_codec_embedding(q, c);
for i in 0..2048 {
step_sum[i] += emb[i];
}
}
audio_pool.push(step_sum);
}
let t_len = text_pool.len();
let a_len = audio_pool.len();
let max_len = t_len.max(a_len);
let mut body = Vec::with_capacity(max_len);
for i in 0..max_len {
let text_vec = if i < t_len { &text_pool[i] } else { tts_pad };
let audio_vec = if i < a_len { &audio_pool[i] } else { tts_pad };
let fused: Vec<f32> = text_vec
.iter()
.zip(audio_vec.iter())
.map(|(t, a)| t + a)
.collect();
body.push(fused);
}
Self::build_core_with_clone_body(
text,
tokenizer,
assets,
Some(lang_id),
Some(spk_emb),
instruct,
body,
)
}
pub fn build_custom_prompt(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
spk_id: usize,
lang_id: usize,
instruct: Option<&str>,
) -> Result<PromptData, String> {
Self::build_core(
text,
tokenizer,
assets,
Some(lang_id),
Some(spk_id),
None,
instruct,
None,
)
}
fn build_core_with_clone_body(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
lang_id: Option<usize>,
spk_emb: Option<&[f32]>,
instruct: Option<&str>,
body: Vec<Vec<f32>>,
) -> PromptData {
let mut embeds = Vec::new();
if let Some(ins) = instruct {
let prefix = vec![151644, 872, 198];
for id in prefix {
embeds.push(assets.get_text_embedding(id));
}
let content_ids = tokenizer.encode(ins);
for id in content_ids {
embeds.push(assets.get_text_embedding(id as usize));
}
let suffix = vec![151645, 198];
for id in suffix {
embeds.push(assets.get_text_embedding(id));
}
}
for id in [151644, 77091, 198] {
embeds.push(assets.get_text_embedding(id));
}
let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER);
if let Some(lid) = lang_id {
let ids = [THINK, THINK_BOS, lid, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
} else {
let ids = [NOTHINK, THINK_BOS, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
}
if let Some(se) = spk_emb {
let sum: Vec<f32> = marker_emb
.iter()
.zip(se.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
embeds.extend(body);
let text_ids = tokenizer.encode(text);
let result_spk_emb = spk_emb
.map(|s| s.to_vec())
.unwrap_or_else(|| vec![0.0; 2048]);
PromptData {
embd: embeds,
text_ids: text_ids.into_iter().collect(),
spk_emb: result_spk_emb,
}
}
#[allow(clippy::too_many_arguments)]
pub fn build_core(
text: &str,
tokenizer: &Tokenizer,
assets: &Assets,
lang_id: Option<usize>,
spk_id: Option<usize>,
spk_emb: Option<&[f32]>,
instruct: Option<&str>,
mid_embeds: Option<Vec<Vec<f32>>>,
) -> Result<PromptData, String> {
let mut embeds = Vec::new();
let text_ids = tokenizer.encode(text);
// Check if text_ids is empty (e.g., only punctuation)
if text_ids.is_empty() {
return Err("Text is empty or contains only unsupported characters".to_string());
}
// Check if text contains only punctuation and whitespace
let text_trimmed: String = text.chars().filter(|c| !c.is_whitespace()).collect();
let only_punctuation = text_trimmed.chars().all(|c| {
matches!(
c,
',' | '。'
| '、'
| ';'
| ':'
| '?'
| '!'
| '"'
| '\''
| '('
| ')'
| '【'
| '】'
| '《'
| '》'
| '…'
| '—'
| '~'
| ','
| '.'
| '!'
| '?'
| ';'
| ':'
| '('
| ')'
| '['
| ']'
| '<'
| '>'
| '-'
| '~'
| '`'
| '@'
| '#'
| '$'
| '%'
| '^'
| '&'
| '*'
| '+'
| '='
| '|'
| '\\'
| '/'
| '{'
| '}'
| '_'
)
});
if only_punctuation && !text_trimmed.is_empty() {
return Err(
"Text contains only punctuation marks, which may cause generation issues"
.to_string(),
);
}
let result_spk_emb = spk_emb
.map(|s| s.to_vec())
.unwrap_or_else(|| vec![0.0; 2048]);
let marker_emb = assets.get_text_embedding(TEXT_AUDIO_MARKER);
// 1. Instruct Block (User)
if let Some(ins) = instruct {
// <|im_start|>user\n
let prefix = vec![151644, 872, 198];
for id in prefix {
embeds.push(assets.get_text_embedding(id));
}
let content_ids = tokenizer.encode(ins);
for id in content_ids {
embeds.push(assets.get_text_embedding(id as usize));
}
// <|im_end|>\n
let suffix = vec![151645, 198];
for id in suffix {
embeds.push(assets.get_text_embedding(id));
}
}
// 2. Role Block (Assistant)
// <|im_start|>assistant\n -> [151644, 77091, 198]
for id in [151644, 77091, 198] {
embeds.push(assets.get_text_embedding(id));
}
// 3. Control Block
if let Some(lid) = lang_id {
// THINK, THINK_BOS, lang_id, THINK_EOS
let ids = [THINK, THINK_BOS, lid, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
} else {
// NOTHINK, THINK_BOS, THINK_EOS
let ids = [NOTHINK, THINK_BOS, THINK_EOS];
for &id in &ids {
let e = assets.get_codec_embedding(0, id as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
}
// Speaker ID/Emb
if let Some(sid) = spk_id {
let e = assets.get_codec_embedding(0, sid as i32);
let sum: Vec<f32> = marker_emb
.iter()
.zip(e.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
} else if let Some(se) = spk_emb {
let sum: Vec<f32> = marker_emb
.iter()
.zip(se.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(sum);
}
// 4. Mid Embeds
if let Some(mids) = mid_embeds {
embeds.extend(mids);
}
// 5. Task Text Block
// BOS_TOKEN + PAD
let pad_0 = assets.get_codec_embedding(0, PAD as i32);
let bos_token_emb = assets.get_text_embedding(BOS_TOKEN);
let bos_sum: Vec<f32> = bos_token_emb
.iter()
.zip(pad_0.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(bos_sum);
for &id in &text_ids {
let t_emb = assets.get_text_embedding(id as usize);
let sum: Vec<f32> = t_emb.iter().zip(pad_0.iter()).map(|(a, b)| a + b).collect();
embeds.push(sum);
}
// EOS_TOKEN + PAD
let eos_token_emb = assets.get_text_embedding(EOS_TOKEN);
let eos_sum: Vec<f32> = eos_token_emb
.iter()
.zip(pad_0.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(eos_sum);
// 6. Activation (BOS)
// Python: assets.text_table[151671] + assets.emb_tables[0][p["BOS"]]
let bos_emb = assets.get_codec_embedding(0, BOS as i32);
let act_sum: Vec<f32> = marker_emb
.iter()
.zip(bos_emb.iter())
.map(|(a, b)| a + b)
.collect();
embeds.push(act_sum);
Ok(PromptData {
embd: embeds,
text_ids: text_ids.into_iter().collect(),
spk_emb: result_spk_emb,
})
}
}
+65
View File
@@ -0,0 +1,65 @@
use hound;
use std::path::Path;
pub struct AudioSample {
pub samples: Vec<f32>,
pub sample_rate: u32,
pub channels: u16,
}
impl AudioSample {
pub fn load_wav(path: impl AsRef<Path>) -> Result<Self, String> {
let mut reader = hound::WavReader::open(path).map_err(|e| e.to_string())?;
let spec = reader.spec();
let samples: Vec<f32> = reader
.samples::<i16>()
.map(|s| s.unwrap_or(0) as f32 / 32768.0) // Normalize i16 to f32
.collect();
Ok(Self {
samples,
sample_rate: spec.sample_rate,
channels: spec.channels,
})
}
pub fn save_wav(&self, path: impl AsRef<Path>) -> Result<(), String> {
let spec = hound::WavSpec {
channels: self.channels,
sample_rate: self.sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer = hound::WavWriter::create(path, spec).map_err(|e| e.to_string())?;
for &sample in &self.samples {
let amp = (sample * 32767.0).clamp(-32768.0, 32767.0) as i16;
writer.write_sample(amp).map_err(|e| e.to_string())?;
}
writer.finalize().map_err(|e| e.to_string())?;
Ok(())
}
pub fn to_wav_bytes(&self) -> Vec<u8> {
let spec = hound::WavSpec {
channels: self.channels,
sample_rate: self.sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut cursor = std::io::Cursor::new(Vec::new());
{
let mut writer = hound::WavWriter::new(&mut cursor, spec).unwrap();
for &sample in &self.samples {
let amp = (sample * 32767.0).clamp(-32768.0, 32767.0) as i16;
writer.write_sample(amp).unwrap();
}
writer.finalize().unwrap();
}
cursor.into_inner()
}
pub fn duration(&self) -> f32 {
self.samples.len() as f32 / self.sample_rate as f32
}
}
+67
View File
@@ -0,0 +1,67 @@
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::Path;
pub const MAGIC: &[u8] = b"TTSC";
pub const VERSION: u32 = 1;
pub fn save_cache(path: &Path, codes: &[i64], emb: &[f32]) -> io::Result<()> {
let mut file = File::create(path)?;
file.write_all(MAGIC)?;
file.write_all(&VERSION.to_le_bytes())?;
file.write_all(&codes.len().to_le_bytes())?;
for &code in codes {
file.write_all(&code.to_le_bytes())?;
}
file.write_all(&emb.len().to_le_bytes())?;
for &val in emb {
file.write_all(&val.to_le_bytes())?;
}
Ok(())
}
pub fn load_cache(path: &Path) -> io::Result<(Vec<i64>, Vec<f32>)> {
let mut file = File::open(path)?;
let mut magic = [0u8; 4];
file.read_exact(&mut magic)?;
if magic != MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Invalid magic bytes",
));
}
let mut version_bytes = [0u8; 4];
file.read_exact(&mut version_bytes)?;
let version = u32::from_le_bytes(version_bytes);
if version != VERSION {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Unsupported version",
));
}
let mut len_bytes = [0u8; 8];
file.read_exact(&mut len_bytes)?;
let codes_len = usize::from_le_bytes(len_bytes);
let mut codes = Vec::with_capacity(codes_len);
for _ in 0..codes_len {
let mut bytes = [0u8; 8];
file.read_exact(&mut bytes)?;
codes.push(i64::from_le_bytes(bytes));
}
file.read_exact(&mut len_bytes)?;
let emb_len = usize::from_le_bytes(len_bytes);
let mut emb = Vec::with_capacity(emb_len);
for _ in 0..emb_len {
let mut bytes = [0u8; 4];
file.read_exact(&mut bytes)?;
emb.push(f32::from_le_bytes(bytes));
}
Ok((codes, emb))
}
+23
View File
@@ -0,0 +1,23 @@
use clap::Parser;
use std::path::PathBuf;
#[derive(Debug, Parser)]
#[command(name = "qwen3-tts")]
pub struct Args {
#[arg(short, long, default_value = "model-base")]
pub model: PathBuf,
#[arg(short, long)]
pub text: Option<String>,
#[arg(short, long)]
pub voice: Option<PathBuf>,
#[arg(short, long, default_value = "chinese")]
pub language: String,
#[arg(short, long, default_value = "0.8")]
pub temperature: f32,
#[arg(short, long)]
pub stream: bool,
#[arg(short, long)]
pub output: Option<PathBuf>,
#[arg(short, long)]
pub verbose: bool,
}
+6
View File
@@ -0,0 +1,6 @@
pub mod audio;
pub mod cache;
pub mod config;
pub mod stream;
pub mod tokenizer;
pub mod voice_file;
+8
View File
@@ -0,0 +1,8 @@
//! Stream module placeholder
pub struct TTSStream;
#[derive(Default)]
pub struct TTSResult {
pub audio: Vec<f32>,
pub text: String,
pub codes: Vec<i64>,
}
+36
View File
@@ -0,0 +1,36 @@
use std::path::Path;
use tokenizers::Tokenizer as HfTokenizer;
pub struct Tokenizer {
inner: HfTokenizer,
}
impl Tokenizer {
pub fn load(model_dir: &Path) -> Result<Self, Box<dyn std::error::Error>> {
let path = model_dir.join("tokenizer").join("tokenizer.json");
println!("Loading tokenizer from: {}", path.display());
let inner =
HfTokenizer::from_file(path).map_err(|e| format!("Failed to load tokenizer: {}", e))?;
Ok(Self { inner })
}
pub fn encode(&self, text: &str) -> Vec<u32> {
match self.inner.encode(text, false) {
Ok(encoding) => encoding.get_ids().to_vec(),
Err(e) => {
eprintln!("Error encoding text '{}': {}", text, e);
Vec::new()
}
}
}
pub fn decode(&self, ids: &[u32]) -> String {
match self.inner.decode(ids, false) {
Ok(s) => s,
Err(e) => {
eprintln!("Error decoding ids: {}", e);
String::new()
}
}
}
}
+62
View File
@@ -0,0 +1,62 @@
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VoiceFile {
/// Reference text corresponding to the audio features
#[serde(default)]
pub ref_text: String,
/// Extracted semantic codes from the Audio Encoder
#[serde(default)]
pub audio_codes: Vec<i64>,
/// Speaker embedding vector from the Speaker Encoder
#[serde(alias = "spk_emb")]
pub speaker_embedding: Vec<f32>,
// Metadata
pub name: Option<String>,
pub gender: Option<String>,
pub age: Option<String>,
pub description: Option<String>,
}
impl VoiceFile {
pub fn new(ref_text: String, audio_codes: Vec<i64>, speaker_embedding: Vec<f32>) -> Self {
Self {
ref_text,
audio_codes,
speaker_embedding,
name: None,
gender: None,
age: None,
description: None,
}
}
pub fn with_metadata(
mut self,
name: Option<String>,
gender: Option<String>,
age: Option<String>,
description: Option<String>,
) -> Self {
self.name = name;
self.gender = gender;
self.age = age;
self.description = description;
self
}
pub fn load(path: impl AsRef<Path>) -> Result<Self, String> {
let file = File::open(path).map_err(|e| e.to_string())?;
let reader = std::io::BufReader::new(file);
serde_json::from_reader(reader).map_err(|e| e.to_string())
}
pub fn save(&self, path: impl AsRef<Path>) -> Result<(), String> {
let file = File::create(path).map_err(|e| e.to_string())?;
let writer = std::io::BufWriter::new(file);
serde_json::to_writer_pretty(writer, self).map_err(|e| e.to_string())
}
}
+80
View File
@@ -0,0 +1,80 @@
// Audio Worklet Processor for real-time PCM playback
class AudioProcessor extends AudioWorkletProcessor {
constructor() {
super();
// 使用环形缓冲区,更高效
this.bufferSize = 24000 * 10; // 10秒缓冲
this.buffer = new Float32Array(this.bufferSize);
this.writeIndex = 0;
this.readIndex = 0;
this.available = 0;
// 淡入淡出参数
this.fadeSamples = 48; // 2ms 淡入淡出(24000 * 0.002 = 48)
this.lastSample = 0;
this.isFirstChunk = true;
// 缓冲区状态
this.totalReceived = 0;
this.totalPlayed = 0;
this.isDraining = false;
this.port.onmessage = (event) => {
const data = event.data;
if (data instanceof Float32Array) {
// 写入环形缓冲区,应用淡入淡出
const fadeLen = Math.min(this.fadeSamples, data.length);
for (let i = 0; i < data.length; i++) {
if (this.available < this.bufferSize) {
let sample = Math.max(-1, Math.min(1, data[i]));
// 淡入:开头几帧从上一个样本平滑过渡
if (i < fadeLen && !this.isFirstChunk) {
const fadeRatio = i / fadeLen;
sample = this.lastSample * (1 - fadeRatio) + sample * fadeRatio;
}
this.buffer[this.writeIndex] = sample;
this.writeIndex = (this.writeIndex + 1) % this.bufferSize;
this.available++;
}
}
// 保存最后一个样本用于下次淡入
this.lastSample = data[data.length - 1];
this.isFirstChunk = false;
this.totalReceived += data.length;
} else if (data === 'drain') {
// 标记为正在排空
this.isDraining = true;
}
};
}
process(inputs, outputs, parameters) {
const output = outputs[0];
const channel = output[0];
for (let i = 0; i < channel.length; i++) {
if (this.available > 0) {
channel[i] = this.buffer[this.readIndex];
this.readIndex = (this.readIndex + 1) % this.bufferSize;
this.available--;
this.totalPlayed++;
} else {
channel[i] = 0;
}
}
// 如果正在排空且缓冲区为空,通知主线程
if (this.isDraining && this.available === 0) {
this.port.postMessage({ type: 'buffer_drained', played: this.totalPlayed, received: this.totalReceived });
this.isDraining = false;
}
return true;
}
}
registerProcessor('audio-processor', AudioProcessor);
+631
View File
@@ -0,0 +1,631 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Qwen3-TTS WebUI</title>
<style>
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
min-height: 100vh;
color: #e0e0e0;
padding: 20px;
}
.container {
max-width: 900px;
margin: 0 auto;
}
h1 {
text-align: center;
margin-bottom: 30px;
color: #00d4ff;
text-shadow: 0 0 20px rgba(0, 212, 255, 0.5);
}
.card {
background: rgba(255, 255, 255, 0.05);
border-radius: 16px;
padding: 24px;
margin-bottom: 20px;
backdrop-filter: blur(10px);
border: 1px solid rgba(255, 255, 255, 0.1);
}
.card h2 {
margin-bottom: 16px;
color: #00d4ff;
font-size: 1.2em;
}
.form-group {
margin-bottom: 16px;
}
label {
display: block;
margin-bottom: 8px;
color: #aaa;
}
textarea {
width: 100%;
min-height: 100px;
padding: 12px;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 8px;
background: rgba(0, 0, 0, 0.3);
color: #fff;
font-size: 16px;
resize: vertical;
}
textarea:focus {
outline: none;
border-color: #00d4ff;
box-shadow: 0 0 10px rgba(0, 212, 255, 0.3);
}
select, input[type="number"] {
width: 100%;
padding: 10px;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 8px;
background: rgba(0, 0, 0, 0.3);
color: #fff;
font-size: 14px;
}
select:focus, input:focus {
outline: none;
border-color: #00d4ff;
}
.row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 16px;
}
.checkbox-group {
display: flex;
align-items: center;
gap: 8px;
}
.checkbox-group input[type="checkbox"] {
width: 20px;
height: 20px;
cursor: pointer;
}
.btn {
padding: 12px 24px;
border: none;
border-radius: 8px;
font-size: 16px;
cursor: pointer;
transition: all 0.3s ease;
font-weight: 600;
}
.btn-primary {
background: linear-gradient(135deg, #00d4ff, #0099cc);
color: #fff;
}
.btn-primary:hover {
transform: translateY(-2px);
box-shadow: 0 5px 20px rgba(0, 212, 255, 0.4);
}
.btn-primary:disabled {
opacity: 0.5;
cursor: not-allowed;
transform: none;
}
.btn-secondary {
background: rgba(255, 255, 255, 0.1);
color: #fff;
border: 1px solid rgba(255, 255, 255, 0.2);
}
.btn-secondary:hover {
background: rgba(255, 255, 255, 0.2);
}
.btn-streaming {
background: linear-gradient(135deg, #ff6b6b, #ee5a5a);
color: #fff;
}
.btn-streaming:hover {
box-shadow: 0 5px 20px rgba(255, 107, 107, 0.4);
}
.btn-group {
display: flex;
gap: 12px;
margin-top: 16px;
flex-wrap: wrap;
}
.status {
padding: 12px;
border-radius: 8px;
margin-top: 16px;
}
.status.info {
background: rgba(0, 212, 255, 0.1);
border: 1px solid rgba(0, 212, 255, 0.3);
}
.status.error {
background: rgba(255, 82, 82, 0.1);
border: 1px solid rgba(255, 82, 82, 0.3);
color: #ff5252;
}
.status.success {
background: rgba(76, 175, 80, 0.1);
border: 1px solid rgba(76, 175, 80, 0.3);
color: #4caf50;
}
.audio-player {
margin-top: 16px;
}
audio {
width: 100%;
margin-top: 8px;
}
.loading {
display: inline-block;
width: 20px;
height: 20px;
border: 2px solid rgba(255, 255, 255, 0.3);
border-radius: 50%;
border-top-color: #00d4ff;
animation: spin 1s ease-in-out infinite;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
.hidden {
display: none;
}
.info-text {
font-size: 12px;
color: #888;
margin-top: 4px;
}
.api-info {
font-family: monospace;
background: rgba(0, 0, 0, 0.3);
padding: 12px;
border-radius: 8px;
font-size: 13px;
overflow-x: auto;
}
.api-info code {
color: #00d4ff;
}
.streaming-indicator {
display: inline-flex;
align-items: center;
gap: 8px;
padding: 8px 16px;
background: rgba(255, 107, 107, 0.2);
border-radius: 20px;
font-size: 14px;
color: #ff6b6b;
}
.streaming-indicator .dot {
width: 8px;
height: 8px;
background: #ff6b6b;
border-radius: 50%;
animation: pulse 1s ease-in-out infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
.mode-switch {
display: flex;
gap: 8px;
margin-bottom: 16px;
}
.mode-btn {
padding: 8px 16px;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 8px;
background: transparent;
color: #aaa;
cursor: pointer;
transition: all 0.3s ease;
}
.mode-btn.active {
background: rgba(0, 212, 255, 0.2);
border-color: #00d4ff;
color: #00d4ff;
}
.mode-btn:hover {
border-color: #00d4ff;
}
</style>
</head>
<body>
<div class="container">
<h1>🎵 Qwen3-TTS WebUI</h1>
<div class="card">
<h2>📝 文本输入</h2>
<div class="form-group">
<label for="text">输入要合成的文本:</label>
<textarea id="text" placeholder="请输入要转换的文本...">你好,这是一个测试。</textarea>
</div>
<div class="form-group">
<label for="instruction">指令风格 (可选):</label>
<input type="text" id="instruction" placeholder="例如: Happy, Sad, Excited, Calm...">
<div class="info-text">添加情感风格指令,留空则无</div>
</div>
</div>
<div class="card">
<h2>⚙️ 参数设置</h2>
<div class="row">
<div class="form-group">
<label for="speaker">说话人</label>
<select id="speaker"></select>
</div>
<div class="form-group">
<label for="temperature">Temperature</label>
<input type="number" id="temperature" value="0.9" min="0" max="2" step="0.1">
<div class="info-text">越高越随机,0 为贪婪采样</div>
</div>
<div class="form-group">
<label for="top_k">Top-K</label>
<input type="number" id="top_k" value="50" min="0" max="100" step="1">
<div class="info-text">0 表示禁用</div>
</div>
<div class="form-group">
<label for="top_p">Top-P</label>
<input type="number" id="top_p" value="1.0" min="0" max="1" step="0.05">
<div class="info-text">1.0 表示禁用</div>
</div>
</div>
<div class="row">
<div class="form-group">
<label for="seed">随机种子</label>
<input type="number" id="seed" placeholder="留空为随机">
<div class="info-text">相同种子产生相同结果</div>
</div>
<div class="form-group">
<label for="maxSteps">最大步数</label>
<input type="number" id="maxSteps" value="512" min="64" max="2048" step="64">
<div class="info-text">生成最大 token 数</div>
</div>
</div>
</div>
<div class="card">
<h2>🎵 生成音频</h2>
<div class="mode-switch">
<button class="mode-btn active" id="modeNormal" onclick="setMode('normal')">普通模式</button>
<button class="mode-btn" id="modeStream" onclick="setMode('stream')">流式播放</button>
</div>
<div class="btn-group">
<button class="btn btn-primary" id="generateBtn" onclick="generateTTS()">
生成音频
</button>
<button class="btn btn-streaming hidden" id="streamBtn" onclick="startStreaming()">
🎧 流式播放
</button>
<button class="btn btn-secondary" id="stopBtn" onclick="stopGeneration()" disabled>
停止
</button>
</div>
<div id="streamingStatus" class="hidden" style="margin-top: 16px;">
<div class="streaming-indicator">
<span class="dot"></span>
<span>正在流式播放...</span>
</div>
<div id="sendProgress" style="margin-top: 12px; padding: 12px; background: rgba(255,255,255,0.05); border-radius: 8px; font-family: monospace; font-size: 14px; line-height: 1.6;">
<div style="color: #888; margin-bottom: 8px;">发送进度:</div>
<div id="sentText" style="color: #4ecdc4; word-break: break-all;"></div>
<div id="pendingText" style="color: #666; word-break: break-all;"></div>
</div>
</div>
<div id="status" class="status info hidden"></div>
<div id="audioContainer" class="audio-player hidden">
<audio id="audioPlayer" controls></audio>
</div>
</div>
<div class="card">
<h2>📖 API 使用说明</h2>
<div class="api-info">
<p><strong>HTTP API (非流式):</strong></p>
<p>POST <code>/api/tts</code></p>
<p>Body: <code>{"text": "你好", "speaker": "vivian"}</code></p>
<br>
<p><strong>WebSocket API (流式):</strong></p>
<p>Connect to <code>/api/tts/stream</code></p>
<p>Send: <code>{"text": "你好", "speaker": "vivian"}</code></p>
<p>Receive: Binary audio data (f32 PCM, 24kHz)</p>
</div>
</div>
</div>
<script>
let currentController = null;
let ws = null;
let audioContext = null;
let audioWorkletNode = null;
let isStreaming = false;
let currentMode = 'normal';
async function loadSpeakers() {
try {
const response = await fetch('/api/speakers');
const data = await response.json();
const select = document.getElementById('speaker');
select.innerHTML = '';
data.speakers.forEach(name => {
const option = document.createElement('option');
option.value = name;
option.textContent = name;
select.appendChild(option);
});
} catch (e) {
console.error('Failed to load speakers:', e);
}
}
function setMode(mode) {
currentMode = mode;
document.getElementById('modeNormal').classList.toggle('active', mode === 'normal');
document.getElementById('modeStream').classList.toggle('active', mode === 'stream');
document.getElementById('generateBtn').classList.toggle('hidden', mode === 'stream');
document.getElementById('streamBtn').classList.toggle('hidden', mode === 'normal');
}
function showStatus(message, type = 'info') {
const status = document.getElementById('status');
status.textContent = message;
status.className = `status ${type}`;
status.classList.remove('hidden');
}
function hideStatus() {
document.getElementById('status').classList.add('hidden');
}
function setGenerating(isGenerating) {
const generateBtn = document.getElementById('generateBtn');
const streamBtn = document.getElementById('streamBtn');
const stopBtn = document.getElementById('stopBtn');
generateBtn.disabled = isGenerating;
streamBtn.disabled = isGenerating;
stopBtn.disabled = !isGenerating;
if (isGenerating) {
generateBtn.innerHTML = '<span class="loading"></span> 生成中...';
streamBtn.innerHTML = '<span class="loading"></span> 连接中...';
} else {
generateBtn.innerHTML = '生成音频';
streamBtn.innerHTML = '🎧 流式播放';
}
}
async function generateTTS() {
const text = document.getElementById('text').value.trim();
if (!text) {
showStatus('请输入文本', 'error');
return;
}
const speaker = document.getElementById('speaker').value;
const temperature = parseFloat(document.getElementById('temperature').value) || 0.9;
const top_k = parseInt(document.getElementById('top_k').value) || 50;
const top_p = parseFloat(document.getElementById('top_p').value) || 1.0;
const seed = document.getElementById('seed').value ? parseInt(document.getElementById('seed').value) : null;
const maxSteps = parseInt(document.getElementById('maxSteps').value) || 512;
const instruction = document.getElementById('instruction').value.trim() || null;
setGenerating(true);
showStatus('正在生成音频...', 'info');
document.getElementById('audioContainer').classList.add('hidden');
currentController = new AbortController();
try {
const response = await fetch('/api/tts', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
text,
speaker,
temperature,
top_k,
top_p,
seed,
max_steps: maxSteps,
instruction,
}),
signal: currentController.signal,
});
const data = await response.json();
if (data.success && data.audio_base64) {
const audioData = Uint8Array.from(atob(data.audio_base64), c => c.charCodeAt(0));
const blob = new Blob([audioData], { type: 'audio/wav' });
const url = URL.createObjectURL(blob);
const audioPlayer = document.getElementById('audioPlayer');
audioPlayer.src = url;
document.getElementById('audioContainer').classList.remove('hidden');
showStatus(`生成成功!时长: ${(data.duration_ms / 1000).toFixed(2)}秒`, 'success');
} else {
showStatus(data.message || '生成失败', 'error');
}
} catch (e) {
if (e.name === 'AbortError') {
showStatus('已停止生成', 'info');
} else {
showStatus(`错误: ${e.message}`, 'error');
}
} finally {
setGenerating(false);
currentController = null;
}
}
async function startStreaming() {
const text = document.getElementById('text').value.trim();
if (!text) {
showStatus('请输入文本', 'error');
return;
}
const speaker = document.getElementById('speaker').value;
const temperature = parseFloat(document.getElementById('temperature').value) || 0.9;
const top_k = parseInt(document.getElementById('top_k').value) || 50;
const top_p = parseFloat(document.getElementById('top_p').value) || 1.0;
const seed = document.getElementById('seed').value ? parseInt(document.getElementById('seed').value) : null;
const instruction = document.getElementById('instruction').value.trim() || null;
setGenerating(true);
document.getElementById('streamingStatus').classList.remove('hidden');
document.getElementById('audioContainer').classList.add('hidden');
hideStatus();
try {
// 创建 AudioContext(需要用户交互后才能播放)
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: 24000
});
// 确保 AudioContext 是运行状态
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
await audioContext.audioWorklet.addModule('/audio-processor.js?v=2');
audioWorkletNode = new AudioWorkletNode(audioContext, 'audio-processor');
audioWorkletNode.connect(audioContext.destination);
// 监听 AudioWorklet 消息
audioWorkletNode.port.onmessage = (event) => {
if (event.data.type === 'prebuffer_ready') {
console.log(`Pre-buffer ready: ${event.data.available} samples (${(event.data.available / 24000).toFixed(2)}s)`);
showStatus(`预缓存就绪,开始播放...`, 'info');
} else if (event.data.type === 'buffer_status') {
const bufferMs = (event.data.available / 24000 * 1000).toFixed(0);
console.log(`Buffer: ${event.data.available} samples (${bufferMs}ms), playing: ${event.data.isPlaying}, draining: ${event.data.isDraining}, drainRequested: ${event.data.drainRequested}, received: ${event.data.totalReceived}, played: ${event.data.totalPlayed}`);
} else if (event.data.type === 'drain_started') {
console.log(`Drain started: available=${event.data.available}, received=${event.data.totalReceived}, played=${event.data.totalPlayed}`);
} else if (event.data.type === 'buffer_drained') {
console.log(`Buffer drained: played ${event.data.played}, received ${event.data.received}`);
if (isStreaming) {
stopStreaming();
showStatus('流式播放完成', 'success');
}
}
};
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
ws = new WebSocket(`${protocol}//${window.location.host}/api/tts/stream`);
ws.binaryType = 'arraybuffer';
// 一次性发送所有文本
document.getElementById('sentText').textContent = text;
document.getElementById('pendingText').textContent = '';
ws.onopen = () => {
// 一次性发送所有文本
ws.send(JSON.stringify({
text: text,
speaker,
temperature,
top_k,
top_p,
seed,
instruction,
}));
// 发送结束信号
ws.send('end');
};
ws.onmessage = (event) => {
if (event.data instanceof ArrayBuffer) {
// 音频数据
const float32Data = new Float32Array(event.data);
console.log('Received audio data:', float32Data.length, 'samples');
audioWorkletNode.port.postMessage(float32Data);
} else if (event.data === 'segment_done') {
// 段完成信号
console.log('Segment done');
} else if (event.data === 'done') {
// 服务端确认结束,通知 AudioWorklet 开始排空
console.log('All done, notifying AudioWorklet to drain...');
ws.close();
ws = null;
// 通知 AudioWorklet 开始排空缓冲区
if (audioWorkletNode) {
audioWorkletNode.port.postMessage('drain');
}
} else {
try {
const data = JSON.parse(event.data);
if (data.error) {
showStatus(`错误: ${data.error}`, 'error');
stopStreaming();
}
} catch (e) {}
}
};
ws.onerror = (error) => {
showStatus('WebSocket 连接错误', 'error');
stopStreaming();
};
ws.onclose = () => {
// WebSocket 关闭时不立即停止播放
// 让 AudioWorklet 继续播放缓冲区中的音频
console.log('WebSocket closed, letting buffer drain...');
};
isStreaming = true;
} catch (e) {
showStatus(`错误: ${e.message}`, 'error');
stopStreaming();
}
}
function stopStreaming() {
isStreaming = false;
if (ws) {
ws.close();
ws = null;
}
if (audioWorkletNode) {
audioWorkletNode.port.postMessage('reset');
audioWorkletNode.disconnect();
audioWorkletNode = null;
}
if (audioContext) {
audioContext.close();
audioContext = null;
}
document.getElementById('streamingStatus').classList.add('hidden');
document.getElementById('sentText').textContent = '';
document.getElementById('pendingText').textContent = '';
setGenerating(false);
}
function stopGeneration() {
if (currentController) {
currentController.abort();
}
if (isStreaming) {
stopStreaming();
}
}
// Initialize
loadSpeakers();
</script>
</body>
</html>