//! AiPlugin — AI 语音对话插件 (M2.1) //! //! 设备本地跑 ASR (whisper.cpp) → LLM (llama.cpp) → TTS (piper) 管线, //! 提供语音/文字对话回合能力。声音一律返回客户端播放,设备不出声。 //! //! # 架构 //! ```text //! HTTP /api/chat → ChatPipeline.run() → ASR → LLM → TTS → ChatResponse //! (Arc 共享, HTTP 在 spawn_blocking 调用) //! ``` //! //! AiPlugin 本身是薄层:持有 ChatPipeline Arc 供 HTTP 层取用, //! 并处理消息系统触发的联动(如 talk/idle 状态切换)。 //! //! # 硬件约束 (Spike 2026-07-03 张明远实测) //! - 测试机全志 A733: 2×A78@2.0G + 6×A55@1.8G, 4G 内存 //! - LLM 推理必须 t=2 锁大核 (全核反而崩且挤死视频) //! - 严禁默认上下文长度 (4096/8192 会吃 1-3.3GB) //! - 推荐默认 Qwen2.5-0.5B Q4_K_M (16.4 t/s, RSS 985M) pub mod backend; pub mod chat; pub mod model_manager; pub use chat::{ChatPipeline, SessionContext}; use crate::core::message::Message; use crate::core::plugin::*; use anyhow::Result; use std::path::PathBuf; use std::sync::{Arc, Mutex}; /// AiPlugin — AI 对话插件(薄层) /// /// 持有 ChatPipeline 的共享句柄,供 HTTP 层取用。 /// 自身处理消息系统触发的联动(talk/idle 状态切换等)。 pub struct AiPlugin { ctx: Option, /// 对话管线(HTTP 层通过 Arc clone 直接调用) pipeline: Arc, /// 工具目录 tools_dir: PathBuf, } impl AiPlugin { /// 创建默认本地后端实例 pub fn new_default( model_store: PathBuf, tools_dir: PathBuf, whisper_lib_dir: Option, piper_lib_dir: Option, piper_config: Option, espeak_data_dir: Option, tmp_dir: PathBuf, context_window: usize, ) -> Self { let mut backend_builder = backend::LocalCliBackend::new(tools_dir.clone()); if let Some(lib_dir) = whisper_lib_dir { backend_builder = backend_builder.with_whisper_lib_dir(lib_dir); } // piper 三个依赖必须同时设置 if let (Some(lib_dir), Some(config), Some(espeak_data)) = (piper_lib_dir, piper_config, espeak_data_dir) { backend_builder = backend_builder.with_piper_deps(lib_dir, config, espeak_data); } let backend = Box::new(backend_builder); let models = model_manager::ModelManager::new(model_store); let pipeline = ChatPipeline::new(backend, models, tmp_dir, context_window); Self { ctx: None, pipeline, tools_dir, } } /// 获取对话管线共享句柄(main.rs 注册时传给 HttpPlugin) pub fn pipeline(&self) -> Arc { Arc::clone(&self.pipeline) } /// 获取模型管理器共享句柄(main.rs 注册时传给 HttpPlugin) pub fn models(&self) -> Arc> { Arc::clone(&self.pipeline.models) } } impl Plugin for AiPlugin { fn id(&self) -> &str { "ai" } fn info(&self) -> PluginInfo { PluginInfo { name: "AiPlugin".to_string(), version: "0.1.0".to_string(), description: "AI 语音对话 (ASR/LLM/TTS 本地推理)".to_string(), platform: Platform::LinuxArm64, } } fn capabilities(&self) -> Vec { vec!["chat".to_string(), "model_management".to_string()] } fn init(&mut self, ctx: PluginContext) -> Result<()> { self.ctx = Some(ctx); Ok(()) } fn start(&mut self) -> Result<()> { // 确保临时目录存在 if let Some(tmp) = self.pipeline.tmp_dir.parent() { std::fs::create_dir_all(tmp).ok(); } std::fs::create_dir_all(&self.pipeline.tmp_dir).ok(); // 预加载默认模型清单 let mut models = self.pipeline.models.lock().unwrap(); if let Err(e) = models.ensure_default_models() { eprintln!("[AiPlugin] 警告: 模型初始化失败: {e}"); } drop(models); println!( "[AiPlugin] 启动 (tools={}, tmp={})", self.tools_dir.display(), self.pipeline.tmp_dir.display() ); Ok(()) } fn handle_message(&mut self, msg: Message) -> Result<()> { match msg { Message::ChatRequest(req) => { // 通过消息系统发起的对话(非 HTTP 路径),同步执行并广播结果 let resp = self.pipeline.run(&req); if let Some(ctx) = &self.ctx { let _ = ctx.tx.send(crate::core::message::Envelope { from: self.id().to_string(), to: crate::core::message::Destination::Broadcast, message: Message::ChatResponse(resp), }); } } Message::Shutdown => { self.stop()?; } Message::StateChanged { old_state, new_state } => { // 画面联动:进入/离开 talk 状态时记录日志(实际联动由 video 插件处理状态机) if new_state == "talk" || old_state == "talk" { println!("[AiPlugin] 画面状态: {old_state} → {new_state}"); } } _ => {} } Ok(()) } fn stop(&mut self) -> Result<()> { // 清理临时文件 if self.pipeline.tmp_dir.exists() { if let Ok(entries) = std::fs::read_dir(&self.pipeline.tmp_dir) { for entry in entries.flatten() { let path = entry.path(); if path.extension().and_then(|s| s.to_str()) == Some("wav") { let _ = std::fs::remove_file(&path); } } } } Ok(()) } }