feat(M2.1): Web 端语音录放闭环

1. 新增 GET /api/chat/audio/file?path=<tmp_dir内路径> 路由
   - 路径白名单:canonicalize 后必须位于 config.ai.tmp_dir 内
   - 返回 audio/wav 二进制流(浏览器可直接 <audio> 播放)
   - 60s 后自动清理临时文件
   - 修复:原 /download/:filename 只能读 downloads_dir,TTS 输出在 tmp_dir 无法访问

2. POST /api/chat/audio 支持 X-Audio-Format 头
   - 浏览器 MediaRecorder 产 webm/ogg/m4a,whisper-cli 靠扩展名判断格式
   - 服务端按头指示决定文件扩展名(webm/ogg/m4a/mp3/wav)

3. chat.html 录音 UI + 客户端 WAV 转码
   - 按住🎙按钮录音(MediaRecorder),松开发送
   - 客户端用 Web Audio API 解码 + 重采样到 16k 单声道 PCM16
   - 手写 WAV header,输出标准 16k mono wav(whisper-cli 直接可处理)
   - 非安全上下文(HTTP)下禁用录音并显示 Chrome flag 启用指引
   - 回复音频改用 /api/chat/audio/file 路径播放
   - 显示 ASR 转写文本(🎤 你说: xxx)
This commit is contained in:
2026-07-06 14:18:17 +08:00
parent 8617a0e708
commit 5df99ced7b
2 changed files with 180 additions and 55 deletions

View File

@@ -7,6 +7,7 @@ use futures_util::{SinkExt, StreamExt, TryStreamExt};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use std::convert::Infallible;
use std::path::{Path, PathBuf};
use std::sync::{mpsc, Arc};
@@ -147,6 +148,7 @@ pub(crate) fn build_routes(
let ai_api = chat_text_route(Arc::clone(&state))
.or(chat_audio_route(Arc::clone(&state)))
.or(chat_audio_file_route(Arc::clone(&state)))
.or(models_list_route(Arc::clone(&state)))
.or(model_download_route(Arc::clone(&state)))
.or(model_switch_route(Arc::clone(&state)))
@@ -2031,7 +2033,11 @@ fn chat_text_route(
})
}
/// POST /api/chat/audio — 语音对话App 主路径,上传 wav
/// POST /api/chat/audio — 语音对话App/Web 主路径,上传 wav/webm/ogg 等浏览器录音
///
/// 注意:上传的音频会被传给 whisper-cli它通过文件扩展名判断格式。
/// 浏览器 MediaRecorder 默认产 webm/opus移动端可能产 m4a/aac。
/// 服务端不强校验格式,但建议客户端尽量转 wav/16k 单声道。
fn chat_audio_route(
state: Arc<HttpState>,
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
@@ -2039,65 +2045,165 @@ fn chat_audio_route(
.and(warp::post())
.and(warp::body::content_length_limit(20 * 1024 * 1024))
.and(warp::body::bytes())
.and(warp::header::optional::<String>("X-Audio-Format"))
.and(with_state(state))
.and_then(|bytes: bytes::Bytes, state: Arc<HttpState>| async move {
let pipeline = match state.ai_pipeline() {
Some(p) => p,
None => {
return Ok::<_, Infallible>(error_json(
StatusCode::SERVICE_UNAVAILABLE,
"AI 插件未就绪",
))
}
};
.and_then(
|bytes: bytes::Bytes,
fmt_hint: Option<String>,
state: Arc<HttpState>| async move {
let pipeline = match state.ai_pipeline() {
Some(p) => p,
None => {
return Ok::<_, Infallible>(error_json(
StatusCode::SERVICE_UNAVAILABLE,
"AI 插件未就绪",
))
}
};
let config = state.config();
let tmp_dir = config.ai.tmp_dir.clone();
let _ = std::fs::create_dir_all(&tmp_dir);
let audio_path = tmp_dir.join(format!(
"asr_{}.wav",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0)
));
if let Err(e) = std::fs::write(&audio_path, &bytes) {
return Ok(error_json(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("保存音频失败: {e}"),
let config = state.config();
let tmp_dir = config.ai.tmp_dir.clone();
let _ = std::fs::create_dir_all(&tmp_dir);
// 根据客户端上报的格式或默认 wav 决定扩展名whisper-cli 靠扩展名判断)
let ext = match fmt_hint.as_deref() {
Some("webm") => "webm",
Some("ogg") => "ogg",
Some("m4a") => "m4a",
Some("mp3") => "mp3",
Some("wav") | _ => "wav",
};
let audio_path = tmp_dir.join(format!(
"asr_{}.{ext}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0)
));
}
if let Err(e) = std::fs::write(&audio_path, &bytes) {
return Ok(error_json(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("保存音频失败: {e}"),
));
}
let session_id = format!("app_{}", std::process::id());
let chat_req = ChatRequest {
session_id,
input: ChatInput::Audio {
path: audio_path.to_string_lossy().into_owned(),
},
persona_prompt: config.character.persona_prompt.clone(),
max_tokens: config.character.max_tokens,
};
let session_id = format!("app_{}", std::process::id());
let chat_req = ChatRequest {
session_id,
input: ChatInput::Audio {
path: audio_path.to_string_lossy().into_owned(),
},
persona_prompt: config.character.persona_prompt.clone(),
max_tokens: config.character.max_tokens,
};
let audio_path_clone = audio_path.clone();
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
.await
.unwrap_or_else(|e| ChatResponse {
session_id: String::new(),
transcription: None,
reply_text: String::new(),
reply_audio_path: None,
error: Some(format!("AI 管线执行失败: {e}")),
});
let audio_path_clone = audio_path.clone();
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
.await
.unwrap_or_else(|e| ChatResponse {
session_id: String::new(),
transcription: None,
reply_text: String::new(),
reply_audio_path: None,
error: Some(format!("AI 管线执行失败: {e}")),
});
// 清理上传的临时音频
let _ = std::fs::remove_file(&audio_path_clone);
// 清理上传的临时音频
let _ = std::fs::remove_file(&audio_path_clone);
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {
Ok(json_response(StatusCode::OK, &resp))
}
})
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {
Ok(json_response(StatusCode::OK, &resp))
}
},
)
}
/// GET /api/chat/audio/file?path=<绝对路径> — 读取 TTS 生成的临时回复音频
///
/// 安全path 必须在 config.ai.tmp_dir 内,且文件存在。返回 wav 二进制流。
fn chat_audio_file_route(
state: Arc<HttpState>,
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
warp::path!("api" / "chat" / "audio" / "file")
.and(warp::get())
.and(warp::query::<HashMap<String, String>>())
.and(with_state(state))
.and_then(
|params: HashMap<String, String>, state: Arc<HttpState>| async move {
let raw_path = match params.get("path") {
Some(p) => p.clone(),
None => {
return Ok::<_, Infallible>(
error_json(StatusCode::BAD_REQUEST, "缺少 path 参数"),
)
}
};
let config = state.config();
let tmp_dir = config.ai.tmp_dir.clone();
// 路径白名单:必须规范化后位于 tmp_dir 内
let candidate = std::path::Path::new(&raw_path);
let canonical_target = match candidate.canonicalize() {
Ok(p) => p,
Err(_) => {
return Ok(error_json(
StatusCode::NOT_FOUND,
"音频文件不存在或无法访问",
))
}
};
let canonical_tmp = match tmp_dir.canonicalize() {
Ok(p) => p,
Err(_) => {
return Ok(error_json(
StatusCode::INTERNAL_SERVER_ERROR,
"tmp_dir 未配置或不存在",
))
}
};
if !canonical_target.starts_with(&canonical_tmp) {
return Ok(error_json(
StatusCode::FORBIDDEN,
"路径越界:只允许访问 tmp_dir 内的音频文件",
));
}
if !canonical_target.is_file() {
return Ok(error_json(StatusCode::NOT_FOUND, "音频文件不存在"));
}
// 返回内联音频(浏览器可直接播放)
match std::fs::read(&canonical_target) {
Ok(data) => {
let cleanup_target = canonical_target.clone();
let _ = std::thread::spawn(move || {
// 60s 后清理(足够浏览器播放完)
std::thread::sleep(std::time::Duration::from_secs(60));
let _ = std::fs::remove_file(cleanup_target);
});
match warp::http::Response::builder()
.header("Content-Type", "audio/wav")
.header("Content-Length", data.len().to_string())
.header("Cache-Control", "no-store")
.body(data)
{
Ok(response) => Ok(response.into_response()),
Err(_) => Ok(warp::reply::with_status(
warp::reply::html("响应构建失败".to_string()),
StatusCode::INTERNAL_SERVER_ERROR,
)
.into_response()),
}
}
Err(e) => Ok(error_json(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("读取音频失败: {e}"),
)),
}
},
)
}
// ── AI 模型管理 API (M2.1) ──