feat(M2.1): AI 语音对话插件骨架 + 模型管理 + HTTP API + config schema 扩展
这是 M2.1（语音数字生命 V1）的后端骨架实现，不含 Web UI / Flutter / 设备部署。 新增文件: - src/plugins/ai/mod.rs: AiPlugin 薄层（持有 ChatPipeline Arc，处理消息系统联动） - src/plugins/ai/backend.rs: AiBackend trait + LocalCliBackend (whisper-cli/llama-cli/piper 子进程) + CloudBackend 占位（V1 返回"暂未支持"） - src/plugins/ai/chat.rs: ChatPipeline 对话管线执行器（HTTP 层 spawn_blocking 直接调用） - src/plugins/ai/model_manager.rs: ModelManager 模型资产管理 (清单/下载/切换/删除/配额/档位守门，参照 plugin_repo + version_manager 模式) 修改文件: - src/core/message.rs: 新增 ChatRequest/ChatResponse/AiModelEvent 消息类型 - src/core/config.rs: AppConfig 新增 character 块（角色元信息+人设）+ ai 块（后端配置） 均为 #[serde(default)] 向后兼容旧配置 - src/plugins/mod.rs: 注册 ai 模块 - src/plugins/http/mod.rs: HttpState 新增 ai_pipeline + ai_models 字段及注册方法; HttpPlugin 新增 set_ai_pipeline/set_ai_models 方法 - src/plugins/http/routes.rs: 新增 6 个 AI 相关路由 - POST /api/chat/text (文字对话，Web 端主路径) - POST /api/chat/audio (语音对话，App 主路径) - GET /api/models (模型清单+水位+配额) - POST /api/models/download (下载模型，后台线程执行) - POST /api/models/switch (切换激活模型) - POST /api/models/delete (删除模型，保护当前激活) - src/main.rs: 注册 AiPlugin，连接 pipeline 到 HttpPlugin 技术决策: - 对话管线用 spawn_blocking 而非消息系统，满足 HTTP 同步响应需求 - ChatPipeline 用 Arc<Mutex> 共享，HTTP 和 AiPlugin 共用同一实例 - 互斥用 try_lock，忙时返回 409 而非阻塞 - Spike 结论固化: LLM t=2 锁大核、ctx=1024 限死、Qwen2.5-0.5B 默认档 待完成 (后续提交): - Web 控制端 UI (文字对话 + 角色切换 + 模型管理界面) - Flutter App (角色页/语音页/模型管理页) - 设备端部署 llama.cpp/whisper.cpp/piper 二进制 + 模型下载 - 画面联动 (talk/idle 状态切换) - 测试 注: Windows 开发环境缺 dbus，无法本地 cargo check；待目标机验证。
This commit is contained in:
213
src/plugins/ai/backend.rs
Normal file
213
src/plugins/ai/backend.rs
Normal file
@@ -0,0 +1,213 @@
|
||||
//! AI 后端 trait + 本地命令行实现
|
||||
//!
|
||||
//! 抽象 ASR/LLM/TTS 三层，支持本地命令行（LocalCliBackend）和未来云端后端。
|
||||
//! V1 只实现 local；cloud 配置时返回"暂未支持"错误。
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
/// AI 后端抽象 trait
|
||||
pub trait AiBackend: Send {
|
||||
/// 语音转文字 (ASR)
|
||||
/// - audio_path: 16kHz mono wav 临时文件
|
||||
/// - model_path: whisper.cpp 模型 (ggml-tiny.bin 等)
|
||||
fn asr(&self, audio_path: &Path, model_path: &Path) -> Result<String>;
|
||||
|
||||
/// LLM 对话
|
||||
/// - model_path: llama.cpp GGUF 模型
|
||||
/// - persona: 角色人设 system prompt
|
||||
/// - user_message: 本轮用户输入
|
||||
/// - history: 历史轮次 (role, content)
|
||||
/// - max_tokens: 最大回复 token (0=默认 128)
|
||||
fn llm_chat(
|
||||
&self,
|
||||
model_path: &Path,
|
||||
persona: &str,
|
||||
user_message: &str,
|
||||
history: &[(String, String)],
|
||||
max_tokens: u16,
|
||||
) -> Result<String>;
|
||||
|
||||
/// 文字转语音 (TTS)
|
||||
/// - text: 要合成的文本
|
||||
/// - model_path: piper 模型目录/文件
|
||||
/// - output_path: 输出 wav 文件路径
|
||||
fn tts(&self, text: &str, model_path: &Path, output_path: &Path) -> Result<()>;
|
||||
|
||||
/// 后端标识（local / cloud）
|
||||
fn backend_name(&self) -> &str;
|
||||
}
|
||||
|
||||
/// 本地命令行后端
|
||||
///
|
||||
/// 通过子进程调用 whisper-cli / llama-cli / piper 二进制。
|
||||
/// Spike (2026-07-03) 验证的命令行参数固化于此。
|
||||
pub struct LocalCliBackend {
|
||||
/// 工具目录（含 whisper-cli, llama-cli, piper 二进制）
|
||||
tools_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl LocalCliBackend {
|
||||
pub fn new(tools_dir: PathBuf) -> Self {
|
||||
Self { tools_dir }
|
||||
}
|
||||
|
||||
fn whisper_cli(&self) -> PathBuf {
|
||||
self.tools_dir.join("whisper-cli")
|
||||
}
|
||||
|
||||
fn llama_cli(&self) -> PathBuf {
|
||||
self.tools_dir.join("llama-cli")
|
||||
}
|
||||
|
||||
fn piper(&self) -> PathBuf {
|
||||
self.tools_dir.join("piper")
|
||||
}
|
||||
}
|
||||
|
||||
impl AiBackend for LocalCliBackend {
|
||||
fn asr(&self, audio_path: &Path, model_path: &Path) -> Result<String> {
|
||||
// whisper-cli -m <model> -f <audio> -l zh --no-timestamps -otxt -of <tmp>
|
||||
// 输出到 <tmp>.txt，读取后返回
|
||||
let tmp_out = audio_path.with_extension("asr.txt");
|
||||
let output = Command::new(self.whisper_cli())
|
||||
.arg("-m")
|
||||
.arg(model_path)
|
||||
.arg("-f")
|
||||
.arg(audio_path)
|
||||
.arg("-l")
|
||||
.arg("zh")
|
||||
.arg("--no-timestamps")
|
||||
.arg("-otxt")
|
||||
.arg("-of")
|
||||
.arg(&tmp_out)
|
||||
.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"whisper-cli 失败: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
let txt_path = tmp_out.with_extension("txt");
|
||||
let text = std::fs::read_to_string(&txt_path)?;
|
||||
let _ = std::fs::remove_file(&txt_path);
|
||||
Ok(text.trim().to_string())
|
||||
}
|
||||
|
||||
fn llm_chat(
|
||||
&self,
|
||||
model_path: &Path,
|
||||
persona: &str,
|
||||
user_message: &str,
|
||||
history: &[(String, String)],
|
||||
max_tokens: u16,
|
||||
) -> Result<String> {
|
||||
// 构造 llama-cli 对话 prompt
|
||||
// 格式参考 spike: llama-cli -m <model> -p <prompt> -n <max_tokens> -t 2 -c 1024 --temp 0.7
|
||||
let mut prompt = String::new();
|
||||
prompt.push_str(&format!("system\n{persona}\n"));
|
||||
for (role, content) in history {
|
||||
prompt.push_str(role);
|
||||
prompt.push('\n');
|
||||
prompt.push_str(content);
|
||||
prompt.push('\n');
|
||||
}
|
||||
prompt.push_str("user\n");
|
||||
prompt.push_str(user_message);
|
||||
prompt.push('\n');
|
||||
prompt.push_str("assistant\n");
|
||||
|
||||
let tokens = if max_tokens == 0 { 128 } else { max_tokens as u32 };
|
||||
|
||||
let output = Command::new(self.llama_cli())
|
||||
.arg("-m")
|
||||
.arg(model_path)
|
||||
.arg("-p")
|
||||
.arg(&prompt)
|
||||
.arg("-n")
|
||||
.arg(tokens.to_string())
|
||||
.arg("-t")
|
||||
.arg("2") // 锁大核 (spike 结论: t=2 最优)
|
||||
.arg("-c")
|
||||
.arg("1024") // 限死上下文 (spike 警告: 默认 ctx 吃 1-3.3G)
|
||||
.arg("--temp")
|
||||
.arg("0.7")
|
||||
.arg("--no-display-prompt")
|
||||
.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"llama-cli 失败: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
// llama-cli 输出到 stdout，取生成部分
|
||||
let reply = String::from_utf8_lossy(&output.stdout);
|
||||
Ok(reply.trim().to_string())
|
||||
}
|
||||
|
||||
fn tts(&self, text: &str, model_path: &Path, output_path: &Path) -> Result<()> {
|
||||
// echo <text> | piper -m <model> -f <output> --output_format wav
|
||||
let mut cmd = Command::new(self.piper());
|
||||
cmd.arg("-m")
|
||||
.arg(model_path)
|
||||
.arg("-f")
|
||||
.arg(output_path)
|
||||
.arg("--output_format")
|
||||
.arg("wav")
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped());
|
||||
|
||||
let mut child = cmd.spawn()?;
|
||||
if let Some(mut stdin) = child.stdin.take() {
|
||||
use std::io::Write;
|
||||
stdin.write_all(text.as_bytes())?;
|
||||
}
|
||||
let output = child.wait_with_output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"piper 失败: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn backend_name(&self) -> &str {
|
||||
"local"
|
||||
}
|
||||
}
|
||||
|
||||
/// 云端后端（V1 仅占位，返回"暂未支持"）
|
||||
pub struct CloudBackend;
|
||||
|
||||
impl AiBackend for CloudBackend {
|
||||
fn asr(&self, _audio_path: &Path, _model_path: &Path) -> Result<String> {
|
||||
bail!("云端 ASR 暂未支持 (V1 仅实现 local)")
|
||||
}
|
||||
|
||||
fn llm_chat(
|
||||
&self,
|
||||
_model_path: &Path,
|
||||
_persona: &str,
|
||||
_user_message: &str,
|
||||
_history: &[(String, String)],
|
||||
_max_tokens: u16,
|
||||
) -> Result<String> {
|
||||
bail!("云端 LLM 暂未支持 (V1 仅实现 local)")
|
||||
}
|
||||
|
||||
fn tts(&self, _text: &str, _model_path: &Path, _output_path: &Path) -> Result<()> {
|
||||
bail!("云端 TTS 暂未支持 (V1 仅实现 local)")
|
||||
}
|
||||
|
||||
fn backend_name(&self) -> &str {
|
||||
"cloud"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user