feat(M2.1): Web 端语音录放闭环
1. 新增 GET /api/chat/audio/file?path=<tmp_dir内路径> 路由
- 路径白名单:canonicalize 后必须位于 config.ai.tmp_dir 内
- 返回 audio/wav 二进制流(浏览器可直接 <audio> 播放)
- 60s 后自动清理临时文件
- 修复:原 /download/:filename 只能读 downloads_dir,TTS 输出在 tmp_dir 无法访问
2. POST /api/chat/audio 支持 X-Audio-Format 头
- 浏览器 MediaRecorder 产 webm/ogg/m4a,whisper-cli 靠扩展名判断格式
- 服务端按头指示决定文件扩展名(webm/ogg/m4a/mp3/wav)
3. chat.html 录音 UI + 客户端 WAV 转码
- 按住🎙按钮录音(MediaRecorder),松开发送
- 客户端用 Web Audio API 解码 + 重采样到 16k 单声道 PCM16
- 手写 WAV header,输出标准 16k mono wav(whisper-cli 直接可处理)
- 非安全上下文(HTTP)下禁用录音并显示 Chrome flag 启用指引
- 回复音频改用 /api/chat/audio/file 路径播放
- 显示 ASR 转写文本(🎤 你说: xxx)
This commit is contained in:
@@ -7,6 +7,7 @@ use futures_util::{SinkExt, StreamExt, TryStreamExt};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{mpsc, Arc};
|
||||
@@ -147,6 +148,7 @@ pub(crate) fn build_routes(
|
||||
|
||||
let ai_api = chat_text_route(Arc::clone(&state))
|
||||
.or(chat_audio_route(Arc::clone(&state)))
|
||||
.or(chat_audio_file_route(Arc::clone(&state)))
|
||||
.or(models_list_route(Arc::clone(&state)))
|
||||
.or(model_download_route(Arc::clone(&state)))
|
||||
.or(model_switch_route(Arc::clone(&state)))
|
||||
@@ -2031,7 +2033,11 @@ fn chat_text_route(
|
||||
})
|
||||
}
|
||||
|
||||
/// POST /api/chat/audio — 语音对话(App 主路径,上传 wav)
|
||||
/// POST /api/chat/audio — 语音对话(App/Web 主路径,上传 wav/webm/ogg 等浏览器录音)
|
||||
///
|
||||
/// 注意:上传的音频会被传给 whisper-cli,它通过文件扩展名判断格式。
|
||||
/// 浏览器 MediaRecorder 默认产 webm/opus,移动端可能产 m4a/aac。
|
||||
/// 服务端不强校验格式,但建议客户端尽量转 wav/16k 单声道。
|
||||
fn chat_audio_route(
|
||||
state: Arc<HttpState>,
|
||||
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
|
||||
@@ -2039,65 +2045,165 @@ fn chat_audio_route(
|
||||
.and(warp::post())
|
||||
.and(warp::body::content_length_limit(20 * 1024 * 1024))
|
||||
.and(warp::body::bytes())
|
||||
.and(warp::header::optional::<String>("X-Audio-Format"))
|
||||
.and(with_state(state))
|
||||
.and_then(|bytes: bytes::Bytes, state: Arc<HttpState>| async move {
|
||||
let pipeline = match state.ai_pipeline() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
return Ok::<_, Infallible>(error_json(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI 插件未就绪",
|
||||
))
|
||||
}
|
||||
};
|
||||
.and_then(
|
||||
|bytes: bytes::Bytes,
|
||||
fmt_hint: Option<String>,
|
||||
state: Arc<HttpState>| async move {
|
||||
let pipeline = match state.ai_pipeline() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
return Ok::<_, Infallible>(error_json(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI 插件未就绪",
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let config = state.config();
|
||||
let tmp_dir = config.ai.tmp_dir.clone();
|
||||
let _ = std::fs::create_dir_all(&tmp_dir);
|
||||
let audio_path = tmp_dir.join(format!(
|
||||
"asr_{}.wav",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis())
|
||||
.unwrap_or(0)
|
||||
));
|
||||
if let Err(e) = std::fs::write(&audio_path, &bytes) {
|
||||
return Ok(error_json(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("保存音频失败: {e}"),
|
||||
let config = state.config();
|
||||
let tmp_dir = config.ai.tmp_dir.clone();
|
||||
let _ = std::fs::create_dir_all(&tmp_dir);
|
||||
|
||||
// 根据客户端上报的格式或默认 wav 决定扩展名(whisper-cli 靠扩展名判断)
|
||||
let ext = match fmt_hint.as_deref() {
|
||||
Some("webm") => "webm",
|
||||
Some("ogg") => "ogg",
|
||||
Some("m4a") => "m4a",
|
||||
Some("mp3") => "mp3",
|
||||
Some("wav") | _ => "wav",
|
||||
};
|
||||
let audio_path = tmp_dir.join(format!(
|
||||
"asr_{}.{ext}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis())
|
||||
.unwrap_or(0)
|
||||
));
|
||||
}
|
||||
if let Err(e) = std::fs::write(&audio_path, &bytes) {
|
||||
return Ok(error_json(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("保存音频失败: {e}"),
|
||||
));
|
||||
}
|
||||
|
||||
let session_id = format!("app_{}", std::process::id());
|
||||
let chat_req = ChatRequest {
|
||||
session_id,
|
||||
input: ChatInput::Audio {
|
||||
path: audio_path.to_string_lossy().into_owned(),
|
||||
},
|
||||
persona_prompt: config.character.persona_prompt.clone(),
|
||||
max_tokens: config.character.max_tokens,
|
||||
};
|
||||
let session_id = format!("app_{}", std::process::id());
|
||||
let chat_req = ChatRequest {
|
||||
session_id,
|
||||
input: ChatInput::Audio {
|
||||
path: audio_path.to_string_lossy().into_owned(),
|
||||
},
|
||||
persona_prompt: config.character.persona_prompt.clone(),
|
||||
max_tokens: config.character.max_tokens,
|
||||
};
|
||||
|
||||
let audio_path_clone = audio_path.clone();
|
||||
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
||||
.await
|
||||
.unwrap_or_else(|e| ChatResponse {
|
||||
session_id: String::new(),
|
||||
transcription: None,
|
||||
reply_text: String::new(),
|
||||
reply_audio_path: None,
|
||||
error: Some(format!("AI 管线执行失败: {e}")),
|
||||
});
|
||||
let audio_path_clone = audio_path.clone();
|
||||
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
||||
.await
|
||||
.unwrap_or_else(|e| ChatResponse {
|
||||
session_id: String::new(),
|
||||
transcription: None,
|
||||
reply_text: String::new(),
|
||||
reply_audio_path: None,
|
||||
error: Some(format!("AI 管线执行失败: {e}")),
|
||||
});
|
||||
|
||||
// 清理上传的临时音频
|
||||
let _ = std::fs::remove_file(&audio_path_clone);
|
||||
// 清理上传的临时音频
|
||||
let _ = std::fs::remove_file(&audio_path_clone);
|
||||
|
||||
if resp.error.is_some() {
|
||||
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
|
||||
} else {
|
||||
Ok(json_response(StatusCode::OK, &resp))
|
||||
}
|
||||
})
|
||||
if resp.error.is_some() {
|
||||
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
|
||||
} else {
|
||||
Ok(json_response(StatusCode::OK, &resp))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// GET /api/chat/audio/file?path=<绝对路径> — 读取 TTS 生成的临时回复音频
|
||||
///
|
||||
/// 安全:path 必须在 config.ai.tmp_dir 内,且文件存在。返回 wav 二进制流。
|
||||
fn chat_audio_file_route(
|
||||
state: Arc<HttpState>,
|
||||
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
|
||||
warp::path!("api" / "chat" / "audio" / "file")
|
||||
.and(warp::get())
|
||||
.and(warp::query::<HashMap<String, String>>())
|
||||
.and(with_state(state))
|
||||
.and_then(
|
||||
|params: HashMap<String, String>, state: Arc<HttpState>| async move {
|
||||
let raw_path = match params.get("path") {
|
||||
Some(p) => p.clone(),
|
||||
None => {
|
||||
return Ok::<_, Infallible>(
|
||||
error_json(StatusCode::BAD_REQUEST, "缺少 path 参数"),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let config = state.config();
|
||||
let tmp_dir = config.ai.tmp_dir.clone();
|
||||
|
||||
// 路径白名单:必须规范化后位于 tmp_dir 内
|
||||
let candidate = std::path::Path::new(&raw_path);
|
||||
let canonical_target = match candidate.canonicalize() {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
return Ok(error_json(
|
||||
StatusCode::NOT_FOUND,
|
||||
"音频文件不存在或无法访问",
|
||||
))
|
||||
}
|
||||
};
|
||||
let canonical_tmp = match tmp_dir.canonicalize() {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
return Ok(error_json(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"tmp_dir 未配置或不存在",
|
||||
))
|
||||
}
|
||||
};
|
||||
if !canonical_target.starts_with(&canonical_tmp) {
|
||||
return Ok(error_json(
|
||||
StatusCode::FORBIDDEN,
|
||||
"路径越界:只允许访问 tmp_dir 内的音频文件",
|
||||
));
|
||||
}
|
||||
if !canonical_target.is_file() {
|
||||
return Ok(error_json(StatusCode::NOT_FOUND, "音频文件不存在"));
|
||||
}
|
||||
|
||||
// 返回内联音频(浏览器可直接播放)
|
||||
match std::fs::read(&canonical_target) {
|
||||
Ok(data) => {
|
||||
let cleanup_target = canonical_target.clone();
|
||||
let _ = std::thread::spawn(move || {
|
||||
// 60s 后清理(足够浏览器播放完)
|
||||
std::thread::sleep(std::time::Duration::from_secs(60));
|
||||
let _ = std::fs::remove_file(cleanup_target);
|
||||
});
|
||||
match warp::http::Response::builder()
|
||||
.header("Content-Type", "audio/wav")
|
||||
.header("Content-Length", data.len().to_string())
|
||||
.header("Cache-Control", "no-store")
|
||||
.body(data)
|
||||
{
|
||||
Ok(response) => Ok(response.into_response()),
|
||||
Err(_) => Ok(warp::reply::with_status(
|
||||
warp::reply::html("响应构建失败".to_string()),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
Err(e) => Ok(error_json(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("读取音频失败: {e}"),
|
||||
)),
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// ── AI 模型管理 API (M2.1) ──
|
||||
|
||||
Reference in New Issue
Block a user