不同角色类型采用不同渲染和对话策略：pet(视频+拟声) / human(视频+自然) / singer(视频+元气) / live2d(Canvas渲染+动效)
@@ -113,7 +123,10 @@ const $=s=>document.querySelector(s);const sessionId='web_'+Date.now();let chatB
const charStrategies={pet:{icon:'🐶',label:'宠物',render:'video',placeholder:'和宠物说说话...'},human:{icon:'🧑',label:'数字人',render:'video',placeholder:'和数字人对话...'},singer:{icon:'🎤',label:'歌姬',render:'video',placeholder:'和歌姬聊天...'},live2d:{icon:'✨',label:'Live2D数字人',render:'live2d',placeholder:'和Live2D数字人对话...'}};
document.querySelectorAll('.tab').forEach(t=>{t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('active'));document.querySelectorAll('.panel').forEach(x=>x.classList.remove('active'));t.classList.add('active');$('#panel-'+t.dataset.tab).classList.add('active');if(t.dataset.tab==='character')loadCharacters();if(t.dataset.tab==='models')loadModels();};});
function applyCharStrategy(char){currentChar=char;const strat=charStrategies[char.character_type||'pet']||charStrategies.pet;$('#charAvatar').textContent=strat.icon;$('#charName').textContent=char.name||'未知角色';const tb=$('#charType');tb.textContent=strat.label;tb.className='type-badge '+(char.character_type||'pet');$('#charRender').textContent=char.render_type||strat.render;const ra=$('#renderArea');const cb=$('#chatBox');if((char.render_type||strat.render)==='live2d'){cb.style.display='none';ra.innerHTML='
✨
Live2D 渲染区
模型: '+(char.live2d_model||'未配置')+'';}else{ra.innerHTML='';cb.style.display='block';}$('#chatInput').placeholder=strat.placeholder;}
-async function sendChat(){const input=$('#chatInput');const text=input.value.trim();if(!text||chatBusy)return;chatBusy=true;$('#sendBtn').disabled=true;$('#chatStatus').textContent='思考中...';$('#chatStatus').className='status-bar busy';input.value='';addMsg('user',text);if(currentChar.render_type!=='live2d'&¤tChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);const botMsg=addMsg('bot','思考中...');try{const res=await fetch('/api/chat/text',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({text,session_id:sessionId})});const data=await res.json();if(data.error){botMsg.querySelector('.bubble').textContent='错误: '+data.error;botMsg.querySelector('.bubble').style.color='var(--red)';$('#chatStatus').textContent='错误: '+data.error;$('#chatStatus').className='status-bar error';}else{botMsg.querySelector('.bubble').textContent=data.reply_text||'(空回复)';if(data.reply_audio_path){const au='/download/'+encodeURIComponent(data.reply_audio_path.replace(/^\//,''));const ad=document.createElement('div');ad.className='audio';ad.innerHTML='
🔊 ';botMsg.appendChild(ad);}$('#chatStatus').textContent='';$('#chatStatus').className='status-bar';const rl=(data.reply_text||'').length;const tm=Math.max(2000,rl*200);setTimeout(()=>{triggerTalkState(false);showLive2dTalking(false);},tm);}}catch(e){botMsg.querySelector('.bubble').textContent='请求失败: '+e.message;botMsg.querySelector('.bubble').style.color='var(--red)';$('#chatStatus').textContent='请求失败: '+e.message;$('#chatStatus').className='status-bar error';triggerTalkState(false);showLive2dTalking(false);}chatBusy=false;$('#sendBtn').disabled=false;$('#chatInput').focus();}
+async function sendChat(){const input=$('#chatInput');const text=input.value.trim();if(!text||chatBusy)return;chatBusy=true;$('#sendBtn').disabled=true;$('#chatStatus').textContent='思考中...';$('#chatStatus').className='status-bar busy';input.value='';addMsg('user',text);if(currentChar.render_type!=='live2d'&¤tChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);const botMsg=addMsg('bot','思考中...');try{const res=await fetch('/api/chat/text',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({text,session_id:sessionId})});const data=await res.json();handleChatResponse(data,botMsg);}catch(e){showChatError(botMsg,'请求失败: '+e.message);}chatBusy=false;$('#sendBtn').disabled=false;$('#chatInput').focus();}
+function audioUrl(path){return '/api/chat/audio/file?path='+encodeURIComponent(path);}
+function handleChatResponse(data,botMsg){if(data.error){showChatError(botMsg,'错误: '+data.error);return;}botMsg.querySelector('.bubble').textContent=data.reply_text||(data.transcription?'(无回复)':'(空回复)');if(data.transcription&&data.transcription.trim()){const tr=document.createElement('div');tr.className='meta';tr.style.color='var(--muted)';tr.textContent='🎤 你说: '+data.transcription;botMsg.querySelector('.bubble').insertAdjacentElement('beforebegin',tr);}if(data.reply_audio_path){const ad=document.createElement('div');ad.className='audio-inline';ad.innerHTML='
🔊 回复语音';botMsg.appendChild(ad);}$('#chatStatus').textContent='';$('#chatStatus').className='status-bar';const rl=(data.reply_text||'').length;const tm=Math.max(2000,rl*200);setTimeout(()=>{triggerTalkState(false);showLive2dTalking(false);},tm);}
+function showChatError(botMsg,msg){botMsg.querySelector('.bubble').textContent=msg;botMsg.querySelector('.bubble').style.color='var(--red)';$('#chatStatus').textContent=msg;$('#chatStatus').className='status-bar error';triggerTalkState(false);showLive2dTalking(false);}
function triggerTalkState(talking){const scene=talking?(currentChar.talk_state||'talk'):'idle';fetch('/api/scene',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({scene})}).catch(()=>{});}
function showLive2dTalking(talking){const c=$('#live2dContainer');if(!c)return;const ex=c.querySelector('.live2d-talking');if(talking&&!ex){const b=document.createElement('div');b.className='live2d-talking';b.textContent='说话中';c.appendChild(b);}else if(!talking&&ex)ex.remove();}
function addMsg(role,text){const box=$('#chatBox');const div=document.createElement('div');div.className='msg '+role;const t=new Date().toLocaleTimeString('zh-CN',{hour12:false});div.innerHTML='
'+t+'
';div.querySelector('.bubble').textContent=text;box.appendChild(div);box.scrollTop=box.scrollHeight;return div;}
@@ -125,6 +138,12 @@ async function loadModels(){try{const res=await fetch('/api/models');const data=
async function downloadModel(id){try{const res=await fetch('/api/models/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model_id:id})});const data=await res.json();alert(data.message||'下载已启动');setTimeout(loadModels,3000);}catch(e){alert('失败');}}
async function switchModel(id,kind){try{const res=await fetch('/api/models/switch',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model_id:id,kind})});const data=await res.json();alert(data.message||'已切换');loadModels();}catch(e){alert('失败');}}
async function deleteModel(id){if(!confirm('删除 '+id+'?'))return;try{const res=await fetch('/api/models/delete',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model_id:id})});const data=await res.json();alert(data.message||'已删除');loadModels();}catch(e){alert('失败');}}
+initMic();
+async function initMic(){const micBtn=$('#micBtn');const tip=$('#recTip');if(!navigator.mediaDevices||!navigator.mediaDevices.getUserMedia){tip.innerHTML='当前浏览器不支持录音。请使用文字对话。';return;}if(!window.isSecureContext){micBtn.disabled=true;tip.innerHTML='⚠️ 录音需要安全上下文（HTTPS 或 localhost）。当前为 HTTP 访问，录音已禁用。
启用方法（桌面 Chrome/Edge）：右键浏览器图标 → 属性 → 在目标末尾加
--unsafely-treat-insecure-origin-as-secure=http://192.168.31.105:5000 → 重启浏览器并访问本页。
或直接用文字对话，回复语音仍可正常播放。';return;}let mediaRecorder=null;let chunks=[];let recStream=null;let recTimer=null;const startRec=async()=>{if(chatBusy)return;try{recStream=await navigator.mediaDevices.getUserMedia({audio:{channelCount:1,sampleRate:16000,echoCancellation:true,noiseSuppression:true}});chunks=[];mediaRecorder=new MediaRecorder(recStream);mediaRecorder.ondataavailable=e=>{if(e.data.size>0)chunks.push(e.data);};mediaRecorder.onstop=()=>{if(recTimer){clearTimeout(recTimer);recTimer=null;}recStream.getTracks().forEach(t=>t.stop());recStream=null;const blob=new Blob(chunks,{type:mediaRecorder.mimeType||'audio/webm'});if(blob.size<1000){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('#chatStatus').textContent='录音过短，已取消';$('#chatStatus').className='status-bar error';return;}sendAudioBlob(blob,mediaRecorder.mimeType||'audio/webm');};mediaRecorder.start();micBtn.classList.add('recording');micBtn.textContent='⏹';$('#chatStatus').textContent='录音中...（松开发送）';$('#chatStatus').className='status-bar busy';recTimer=setTimeout(()=>{if(mediaRecorder&&mediaRecorder.state==='recording'){stopRec();$('#chatStatus').textContent='录音达到 30s 上限，自动发送';}},30000);}catch(e){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('#chatStatus').textContent='麦克风启动失败: '+e.message;$('#chatStatus').className='status-bar error';}};const stopRec=()=>{if(mediaRecorder&&mediaRecorder.state==='recording'){mediaRecorder.stop();micBtn.classList.remove('recording');micBtn.textContent='🎙';}};micBtn.disabled=false;micBtn.addEventListener('mousedown',startRec);micBtn.addEventListener('touchstart',e=>{e.preventDefault();startRec();});micBtn.addEventListener('mouseup',stopRec);micBtn.addEventListener('mouseleave',stopRec);micBtn.addEventListener('touchend',e=>{e.preventDefault();stopRec();});}
+async function sendAudioBlob(blob,mimeType){if(chatBusy)return;chatBusy=true;$('#sendBtn').disabled=true;const micBtn=$('#micBtn');micBtn.disabled=true;$('#chatStatus').textContent='转码中...';$('#chatStatus').className='status-bar busy';const userMsg=addMsg('user','🎤 [语音消息]');const playUrl=URL.createObjectURL(blob);const up=document.createElement('div');up.className='audio-inline';up.innerHTML='
你的录音';userMsg.appendChild(up);if(currentChar.render_type!=='live2d'&¤tChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);const botMsg=addMsg('bot','思考中...');try{const wavBlob=await blobToWav(blob);$('#chatStatus').textContent='识别中...';const res=await fetch('/api/chat/audio',{method:'POST',headers:{'Content-Type':'audio/wav','X-Audio-Format':'wav'},body:wavBlob});const data=await res.json();handleChatResponse(data,botMsg);}catch(e){showChatError(botMsg,'请求失败: '+e.message);}chatBusy=false;$('#sendBtn').disabled=false;micBtn.disabled=false;}
+async function blobToWav(blob){const arrayBuf=await blob.arrayBuffer();const audioCtx=new(window.AudioContext||window.webkitAudioContext)({sampleRate:16000});const decoded=await audioCtx.decodeAudioData(arrayBuf);const pcm=decodeToMonoPCM16(decoded,16000);audioCtx.close();return encodeWavBlob(pcm,16000);}
+function decodeToMonoPCM16(audioBuffer,targetRate){const numCh=audioBuffer.numberOfChannels;const len=audioBuffer.length;const srcRate=audioBuffer.sampleRate;const outLen=Math.round(len*targetRate/srcRate);const out=new Int16Array(outLen);const tmp=new Float32Array(outLen);for(let ch=0;ch
{for(let i=0;i
diff --git a/src/plugins/http/routes.rs b/src/plugins/http/routes.rs
index f8cacb7..1a36340 100644
--- a/src/plugins/http/routes.rs
+++ b/src/plugins/http/routes.rs
@@ -7,6 +7,7 @@ use futures_util::{SinkExt, StreamExt, TryStreamExt};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use std::collections::HashMap;
use std::convert::Infallible;
use std::path::{Path, PathBuf};
use std::sync::{mpsc, Arc};
@@ -147,6 +148,7 @@ pub(crate) fn build_routes(
let ai_api = chat_text_route(Arc::clone(&state))
.or(chat_audio_route(Arc::clone(&state)))
+ .or(chat_audio_file_route(Arc::clone(&state)))
.or(models_list_route(Arc::clone(&state)))
.or(model_download_route(Arc::clone(&state)))
.or(model_switch_route(Arc::clone(&state)))
@@ -2031,7 +2033,11 @@ fn chat_text_route(
})
}
-/// POST /api/chat/audio — 语音对话（App 主路径，上传 wav）
+/// POST /api/chat/audio — 语音对话（App/Web 主路径，上传 wav/webm/ogg 等浏览器录音）
+///
+/// 注意：上传的音频会被传给 whisper-cli，它通过文件扩展名判断格式。
+/// 浏览器 MediaRecorder 默认产 webm/opus，移动端可能产 m4a/aac。
+/// 服务端不强校验格式，但建议客户端尽量转 wav/16k 单声道。
fn chat_audio_route(
state: Arc,
) -> impl Filter + Clone {
@@ -2039,65 +2045,165 @@ fn chat_audio_route(
.and(warp::post())
.and(warp::body::content_length_limit(20 * 1024 * 1024))
.and(warp::body::bytes())
+ .and(warp::header::optional::("X-Audio-Format"))
.and(with_state(state))
- .and_then(|bytes: bytes::Bytes, state: Arc| async move {
- let pipeline = match state.ai_pipeline() {
- Some(p) => p,
- None => {
- return Ok::<_, Infallible>(error_json(
- StatusCode::SERVICE_UNAVAILABLE,
- "AI 插件未就绪",
- ))
- }
- };
+ .and_then(
+ |bytes: bytes::Bytes,
+ fmt_hint: Option,
+ state: Arc| async move {
+ let pipeline = match state.ai_pipeline() {
+ Some(p) => p,
+ None => {
+ return Ok::<_, Infallible>(error_json(
+ StatusCode::SERVICE_UNAVAILABLE,
+ "AI 插件未就绪",
+ ))
+ }
+ };
- let config = state.config();
- let tmp_dir = config.ai.tmp_dir.clone();
- let _ = std::fs::create_dir_all(&tmp_dir);
- let audio_path = tmp_dir.join(format!(
- "asr_{}.wav",
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .map(|d| d.as_millis())
- .unwrap_or(0)
- ));
- if let Err(e) = std::fs::write(&audio_path, &bytes) {
- return Ok(error_json(
- StatusCode::INTERNAL_SERVER_ERROR,
- &format!("保存音频失败: {e}"),
+ let config = state.config();
+ let tmp_dir = config.ai.tmp_dir.clone();
+ let _ = std::fs::create_dir_all(&tmp_dir);
+
+ // 根据客户端上报的格式或默认 wav 决定扩展名（whisper-cli 靠扩展名判断）
+ let ext = match fmt_hint.as_deref() {
+ Some("webm") => "webm",
+ Some("ogg") => "ogg",
+ Some("m4a") => "m4a",
+ Some("mp3") => "mp3",
+ Some("wav") | _ => "wav",
+ };
+ let audio_path = tmp_dir.join(format!(
+ "asr_{}.{ext}",
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_millis())
+ .unwrap_or(0)
));
- }
+ if let Err(e) = std::fs::write(&audio_path, &bytes) {
+ return Ok(error_json(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ &format!("保存音频失败: {e}"),
+ ));
+ }
- let session_id = format!("app_{}", std::process::id());
- let chat_req = ChatRequest {
- session_id,
- input: ChatInput::Audio {
- path: audio_path.to_string_lossy().into_owned(),
- },
- persona_prompt: config.character.persona_prompt.clone(),
- max_tokens: config.character.max_tokens,
- };
+ let session_id = format!("app_{}", std::process::id());
+ let chat_req = ChatRequest {
+ session_id,
+ input: ChatInput::Audio {
+ path: audio_path.to_string_lossy().into_owned(),
+ },
+ persona_prompt: config.character.persona_prompt.clone(),
+ max_tokens: config.character.max_tokens,
+ };
- let audio_path_clone = audio_path.clone();
- let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
- .await
- .unwrap_or_else(|e| ChatResponse {
- session_id: String::new(),
- transcription: None,
- reply_text: String::new(),
- reply_audio_path: None,
- error: Some(format!("AI 管线执行失败: {e}")),
- });
+ let audio_path_clone = audio_path.clone();
+ let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
+ .await
+ .unwrap_or_else(|e| ChatResponse {
+ session_id: String::new(),
+ transcription: None,
+ reply_text: String::new(),
+ reply_audio_path: None,
+ error: Some(format!("AI 管线执行失败: {e}")),
+ });
- // 清理上传的临时音频
- let _ = std::fs::remove_file(&audio_path_clone);
+ // 清理上传的临时音频
+ let _ = std::fs::remove_file(&audio_path_clone);
- if resp.error.is_some() {
- Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
- } else {
- Ok(json_response(StatusCode::OK, &resp))
- }
- })
+ if resp.error.is_some() {
+ Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
+ } else {
+ Ok(json_response(StatusCode::OK, &resp))
+ }
+ },
+ )
+}
+
+/// GET /api/chat/audio/file?path=<绝对路径> — 读取 TTS 生成的临时回复音频
+///
+/// 安全：path 必须在 config.ai.tmp_dir 内，且文件存在。返回 wav 二进制流。
+fn chat_audio_file_route(
+ state: Arc,
+) -> impl Filter + Clone {
+ warp::path!("api" / "chat" / "audio" / "file")
+ .and(warp::get())
+ .and(warp::query::>())
+ .and(with_state(state))
+ .and_then(
+ |params: HashMap, state: Arc| async move {
+ let raw_path = match params.get("path") {
+ Some(p) => p.clone(),
+ None => {
+ return Ok::<_, Infallible>(
+ error_json(StatusCode::BAD_REQUEST, "缺少 path 参数"),
+ )
+ }
+ };
+
+ let config = state.config();
+ let tmp_dir = config.ai.tmp_dir.clone();
+
+ // 路径白名单：必须规范化后位于 tmp_dir 内
+ let candidate = std::path::Path::new(&raw_path);
+ let canonical_target = match candidate.canonicalize() {
+ Ok(p) => p,
+ Err(_) => {
+ return Ok(error_json(
+ StatusCode::NOT_FOUND,
+ "音频文件不存在或无法访问",
+ ))
+ }
+ };
+ let canonical_tmp = match tmp_dir.canonicalize() {
+ Ok(p) => p,
+ Err(_) => {
+ return Ok(error_json(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "tmp_dir 未配置或不存在",
+ ))
+ }
+ };
+ if !canonical_target.starts_with(&canonical_tmp) {
+ return Ok(error_json(
+ StatusCode::FORBIDDEN,
+ "路径越界：只允许访问 tmp_dir 内的音频文件",
+ ));
+ }
+ if !canonical_target.is_file() {
+ return Ok(error_json(StatusCode::NOT_FOUND, "音频文件不存在"));
+ }
+
+ // 返回内联音频（浏览器可直接播放）
+ match std::fs::read(&canonical_target) {
+ Ok(data) => {
+ let cleanup_target = canonical_target.clone();
+ let _ = std::thread::spawn(move || {
+ // 60s 后清理（足够浏览器播放完）
+ std::thread::sleep(std::time::Duration::from_secs(60));
+ let _ = std::fs::remove_file(cleanup_target);
+ });
+ match warp::http::Response::builder()
+ .header("Content-Type", "audio/wav")
+ .header("Content-Length", data.len().to_string())
+ .header("Cache-Control", "no-store")
+ .body(data)
+ {
+ Ok(response) => Ok(response.into_response()),
+ Err(_) => Ok(warp::reply::with_status(
+ warp::reply::html("响应构建失败".to_string()),
+ StatusCode::INTERNAL_SERVER_ERROR,
+ )
+ .into_response()),
+ }
+ }
+ Err(e) => Ok(error_json(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ &format!("读取音频失败: {e}"),
+ )),
+ }
+ },
+ )
}
// ── AI 模型管理 API (M2.1) ──