feat: M2.1 客户端骨架、BLE 修复、仓库整理与现状板
- BLE: 修复 BlueZ LocalName 与 Includes 冲突，串行注册与退避 - HTTP: 语音对话补 Live2D talking 标志与 session；Web 补 X-Session-Id - Flutter: 角色/对话/模型页 + API，Gradle/依赖升级，麦克风权限 - 文档: 新增 docs/STATUS.md，校准 CLAUDE/PROGRESS/README - 清理: 移除根目录截图与临时模型，运维脚本迁入 scripts/device
This commit is contained in:
@@ -250,6 +250,11 @@ pub fn run_ble_service(
|
||||
shared: shared.clone(),
|
||||
},
|
||||
);
|
||||
// BlueZ LEAdvertisement1 约束（5.55+）：
|
||||
// Includes 里若含 "local-name"，就禁止再设 LocalName 属性，否则
|
||||
// parse_local_name 报 "Local name already included" →
|
||||
// "Failed to parse advertisement"。
|
||||
// 我们需要自定义设备名，因此只用 LocalName，Includes 仅保留 tx-power。
|
||||
cr.insert(
|
||||
ADV_PATH,
|
||||
&[advertisement_iface],
|
||||
@@ -257,7 +262,7 @@ pub fn run_ble_service(
|
||||
advertisement_type: "peripheral".to_string(),
|
||||
service_uuids: vec![SERVICE_UUID.to_string()],
|
||||
local_name: device_name.clone(),
|
||||
includes: vec!["tx-power".to_string(), "local-name".to_string()],
|
||||
includes: vec!["tx-power".to_string()],
|
||||
},
|
||||
);
|
||||
|
||||
@@ -315,20 +320,37 @@ pub fn run_ble_service(
|
||||
|
||||
// 先尝试清理上一次进程残留的注册（防止崩溃后 BlueZ 状态残留）
|
||||
let _ = unregister_ble_objects(&conn, &adapter_path);
|
||||
// 给 BlueZ 一点时间释放残留 GATT/Adv 实例
|
||||
for _ in 0..5 {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
conn.process(Duration::from_millis(50)).ok();
|
||||
}
|
||||
|
||||
// 串行注册：先 GATT，再 Advertisement。并行时若 Adv 成功、GATT 失败，
|
||||
// 残留 ActiveInstances 会导致后续 "Failed to create entry in database"。
|
||||
let gatt_serial = send_register_gatt_app(&conn, &adapter_path)?;
|
||||
let ad_serial = send_register_advertisement(&conn, &adapter_path)?;
|
||||
if let Ok(mut replies) = registration_replies.lock() {
|
||||
replies.gatt_serial = Some(gatt_serial);
|
||||
}
|
||||
eprintln!("[BLE] RegisterApplication sent, waiting...");
|
||||
if let Err(error) =
|
||||
wait_for_one_registration_reply(&conn, ®istration_replies, "gatt", Duration::from_secs(10))
|
||||
{
|
||||
let _ = unregister_ble_objects(&conn, &adapter_path);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let ad_serial = send_register_advertisement(&conn, &adapter_path)?;
|
||||
if let Ok(mut replies) = registration_replies.lock() {
|
||||
replies.advertisement_serial = Some(ad_serial);
|
||||
}
|
||||
eprintln!("[BLE] registration requests sent, waiting for BlueZ replies...");
|
||||
|
||||
if let Err(error) = wait_for_registration_replies(
|
||||
eprintln!("[BLE] RegisterAdvertisement sent, waiting...");
|
||||
if let Err(error) = wait_for_one_registration_reply(
|
||||
&conn,
|
||||
®istration_replies,
|
||||
gatt_serial,
|
||||
ad_serial,
|
||||
"advertisement",
|
||||
Duration::from_secs(10),
|
||||
) {
|
||||
let _ = unregister_ble_objects(&conn, &adapter_path);
|
||||
@@ -575,36 +597,34 @@ fn replies_for_serial(
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_for_registration_replies(
|
||||
fn wait_for_one_registration_reply(
|
||||
conn: &Connection,
|
||||
replies: &Arc<Mutex<RegistrationReplies>>,
|
||||
gatt_serial: u32,
|
||||
advertisement_serial: u32,
|
||||
which: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<()> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
let op = match which {
|
||||
"gatt" => "RegisterApplication",
|
||||
"advertisement" => "RegisterAdvertisement",
|
||||
other => other,
|
||||
};
|
||||
|
||||
loop {
|
||||
if let Ok(mut replies) = replies.lock() {
|
||||
match reply_status(&mut replies.gatt, "RegisterApplication")? {
|
||||
Some(()) => {}
|
||||
None => {}
|
||||
}
|
||||
match reply_status(&mut replies.advertisement, "RegisterAdvertisement")? {
|
||||
Some(()) => {}
|
||||
None => {}
|
||||
}
|
||||
|
||||
if replies.gatt.is_some() && replies.advertisement.is_some() {
|
||||
let slot = match which {
|
||||
"gatt" => &mut replies.gatt,
|
||||
"advertisement" => &mut replies.advertisement,
|
||||
_ => return Err(anyhow!("unknown BLE registration target: {which}")),
|
||||
};
|
||||
if let Some(()) = reply_status(slot, op)? {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return Err(anyhow!(
|
||||
"timed out waiting for BLE registration reply (gatt_serial={gatt_serial}, advertisement_serial={advertisement_serial})"
|
||||
));
|
||||
return Err(anyhow!("timed out waiting for BLE {op} reply"));
|
||||
}
|
||||
|
||||
conn.process(Duration::from_millis(100).min(deadline.saturating_duration_since(now)))
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! BlePlugin — BLE 配网服务
|
||||
//!
|
||||
//! 通过 D-Bus 与 BlueZ 交互，注册 GATT 服务和 LE Advertisement。
|
||||
//! 含 LocalName 双连接修复。
|
||||
//! 注意：Advertisement 不可同时 Includes["local-name"] + LocalName 属性
|
||||
//! （BlueZ 会报 Failed to parse advertisement）。
|
||||
|
||||
mod gatt;
|
||||
|
||||
@@ -95,11 +96,20 @@ impl Plugin for BlePlugin {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
// BlueZ 残留状态（database / advertisement）需要更长冷却时间
|
||||
let err_text = format!("{error:#}");
|
||||
let wait_secs = if err_text.contains("database")
|
||||
|| err_text.contains("parse advertisement")
|
||||
|| err_text.contains("Already Exists")
|
||||
{
|
||||
8
|
||||
} else {
|
||||
3
|
||||
};
|
||||
eprintln!(
|
||||
"[BlePlugin] worker error: {error:#}, restarting in 3 seconds..."
|
||||
"[BlePlugin] worker error: {err_text}, restarting in {wait_secs} seconds..."
|
||||
);
|
||||
// 等待 3 秒，但如果 stop 被设置则立即退出
|
||||
for _ in 0..30 {
|
||||
for _ in 0..(wait_secs * 10) {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ async function switchModel(id,kind){try{var res=await fetch('/api/models/switch'
|
||||
async function deleteModel(id){if(!confirm('删除 '+id+'?'))return;try{var res=await fetch('/api/models/delete',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model_id:id})});var data=await res.json();toast(data.message||'已删除');loadModels()}catch(e){toast('失败',true)}}
|
||||
initMic();
|
||||
async function initMic(){var micBtn=$('micBtn');var tip=$('recTip');if(!navigator.mediaDevices||!navigator.mediaDevices.getUserMedia){tip.innerHTML='当前浏览器不支持录音。请使用文字对话。';return}if(!window.isSecureContext){micBtn.disabled=true;tip.innerHTML='⚠️ 录音需要安全上下文（HTTPS 或 localhost）。当前为 HTTP 访问，录音已禁用。<br>启用方法（桌面 Chrome/Edge）：右键浏览器图标 → 属性 → 在目标末尾加 <code>--unsafely-treat-insecure-origin-as-secure=http://192.168.31.105:5000</code> → 重启浏览器并访问本页。<br>或直接用文字对话，回复语音仍可正常播放。';return}var mediaRecorder=null,chunks=[],recStream=null,recTimer=null;var startRec=async function(){if(chatBusy)return;try{recStream=await navigator.mediaDevices.getUserMedia({audio:{channelCount:1,sampleRate:16000,echoCancellation:true,noiseSuppression:true}});chunks=[];mediaRecorder=new MediaRecorder(recStream);mediaRecorder.ondataavailable=function(e){if(e.data.size>0)chunks.push(e.data)};mediaRecorder.onstop=function(){if(recTimer){clearTimeout(recTimer);recTimer=null}recStream.getTracks().forEach(function(t){t.stop()});recStream=null;var blob=new Blob(chunks,{type:mediaRecorder.mimeType||'audio/webm'});if(blob.size<1000){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('chatStatus').textContent='录音过短，已取消';$('chatStatus').className='status-bar error';return}sendAudioBlob(blob,mediaRecorder.mimeType||'audio/webm')};mediaRecorder.start();micBtn.classList.add('recording');micBtn.textContent='⏹';$('chatStatus').textContent='录音中...（松开发送）';$('chatStatus').className='status-bar busy';recTimer=setTimeout(function(){if(mediaRecorder&&mediaRecorder.state==='recording'){stopRec();$('chatStatus').textContent='录音达到 30s 上限，自动发送'}},30000)}catch(e){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('chatStatus').textContent='麦克风启动失败: '+e.message;$('chatStatus').className='status-bar error'}};var stopRec=function(){if(mediaRecorder&&mediaRecorder.state==='recording'){mediaRecorder.stop();micBtn.classList.remove('recording');micBtn.textContent='🎙'}};micBtn.disabled=false;micBtn.addEventListener('mousedown',startRec);micBtn.addEventListener('touchstart',function(e){e.preventDefault();startRec()});micBtn.addEventListener('mouseup',stopRec);micBtn.addEventListener('mouseleave',stopRec);micBtn.addEventListener('touchend',function(e){e.preventDefault();stopRec()})}
|
||||
async function sendAudioBlob(blob,mimeType){if(chatBusy)return;chatBusy=true;$('sendBtn').disabled=true;var micBtn=$('micBtn');micBtn.disabled=true;$('chatStatus').textContent='转码中...';$('chatStatus').className='status-bar busy';var userMsg=addMsg('user','🎤 [语音消息]');var playUrl=URL.createObjectURL(blob);var up=document.createElement('div');up.className='audio-inline';up.innerHTML='<span style="font-size:11px;color:var(--muted)">你的录音</span><audio controls src="'+playUrl+'"></audio>';userMsg.appendChild(up);if(currentChar.render_type!=='live2d'&¤tChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);var botMsg=addMsg('bot','思考中...');try{var wavBlob=await blobToWav(blob);$('chatStatus').textContent='识别中...';var res=await fetch('/api/chat/audio',{method:'POST',headers:{'Content-Type':'audio/wav','X-Audio-Format':'wav'},body:wavBlob});var data=await res.json();handleChatResponse(data,botMsg)}catch(e){showChatError(botMsg,'请求失败: '+e.message)}chatBusy=false;$('sendBtn').disabled=false;micBtn.disabled=false}
|
||||
async function sendAudioBlob(blob,mimeType){if(chatBusy)return;chatBusy=true;$('sendBtn').disabled=true;var micBtn=$('micBtn');micBtn.disabled=true;$('chatStatus').textContent='转码中...';$('chatStatus').className='status-bar busy';var userMsg=addMsg('user','🎤 [语音消息]');var playUrl=URL.createObjectURL(blob);var up=document.createElement('div');up.className='audio-inline';up.innerHTML='<span style="font-size:11px;color:var(--muted)">你的录音</span><audio controls src="'+playUrl+'"></audio>';userMsg.appendChild(up);if(currentChar.render_type!=='live2d'&¤tChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);var botMsg=addMsg('bot','思考中...');try{var wavBlob=await blobToWav(blob);$('chatStatus').textContent='识别中...';var res=await fetch('/api/chat/audio',{method:'POST',headers:{'Content-Type':'audio/wav','X-Audio-Format':'wav','X-Session-Id':sessionId},body:wavBlob});var data=await res.json();handleChatResponse(data,botMsg)}catch(e){showChatError(botMsg,'请求失败: '+e.message)}chatBusy=false;$('sendBtn').disabled=false;micBtn.disabled=false}
|
||||
/* WAV 转码：浏览器录音(webm/opus) → 16k mono PCM16 wav，whisper-cli 直接可处理 */
|
||||
async function blobToWav(blob){var arrayBuf=await blob.arrayBuffer();var audioCtx=new(window.AudioContext||window.webkitAudioContext)({sampleRate:16000});var decoded=await audioCtx.decodeAudioData(arrayBuf);var pcm=decodeToMonoPCM16(decoded,16000);audioCtx.close();return encodeWavBlob(pcm,16000)}
|
||||
function decodeToMonoPCM16(audioBuffer,targetRate){var numCh=audioBuffer.numberOfChannels;var len=audioBuffer.length;var srcRate=audioBuffer.sampleRate;var outLen=Math.round(len*targetRate/srcRate);var out=new Int16Array(outLen);var tmp=new Float32Array(outLen);for(var ch=0;ch<numCh;ch++){var data=audioBuffer.getChannelData(ch);for(var i=0;i<outLen;i++){var srcIdx=i*srcRate/targetRate;var i0=Math.floor(srcIdx);var i1=Math.min(i0+1,len-1);var frac=srcIdx-i0;tmp[i]+=data[i0]*(1-frac)+data[i1]*frac}}for(var i=0;i<outLen;i++){var s=tmp[i]/numCh;s=Math.max(-1,Math.min(1,s));out[i]=s<0?s*0x8000:s*0x7FFF}return out}
|
||||
|
||||
@@ -2211,6 +2211,10 @@ fn chat_text_route(
|
||||
/// 注意：上传的音频会被传给 whisper-cli，它通过文件扩展名判断格式。
|
||||
/// 浏览器 MediaRecorder 默认产 webm/opus，移动端可能产 m4a/aac。
|
||||
/// 服务端不强校验格式，但建议客户端尽量转 wav/16k 单声道。
|
||||
///
|
||||
/// 可选头：
|
||||
/// - `X-Audio-Format`: webm|ogg|m4a|mp3|wav
|
||||
/// - `X-Session-Id`: 会话 ID（多端一致上下文）
|
||||
fn chat_audio_route(
|
||||
state: Arc<HttpState>,
|
||||
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
|
||||
@@ -2219,10 +2223,12 @@ fn chat_audio_route(
|
||||
.and(warp::body::content_length_limit(20 * 1024 * 1024))
|
||||
.and(warp::body::bytes())
|
||||
.and(warp::header::optional::<String>("X-Audio-Format"))
|
||||
.and(warp::header::optional::<String>("X-Session-Id"))
|
||||
.and(with_state(state))
|
||||
.and_then(
|
||||
|bytes: bytes::Bytes,
|
||||
fmt_hint: Option<String>,
|
||||
session_hint: Option<String>,
|
||||
state: Arc<HttpState>| async move {
|
||||
let pipeline = match state.ai_pipeline() {
|
||||
Some(p) => p,
|
||||
@@ -2260,7 +2266,9 @@ fn chat_audio_route(
|
||||
));
|
||||
}
|
||||
|
||||
let session_id = format!("app_{}", std::process::id());
|
||||
let session_id = session_hint
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| format!("app_{}", std::process::id()));
|
||||
let chat_req = ChatRequest {
|
||||
session_id,
|
||||
input: ChatInput::Audio {
|
||||
@@ -2270,6 +2278,10 @@ fn chat_audio_route(
|
||||
max_tokens: config.character.max_tokens,
|
||||
};
|
||||
|
||||
// 与文字对话一致：驱动 Live2D 说话态
|
||||
let talking_state = Arc::clone(&state);
|
||||
talking_state.set_live2d_talking(true);
|
||||
|
||||
let audio_path_clone = audio_path.clone();
|
||||
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
||||
.await
|
||||
@@ -2284,6 +2296,21 @@ fn chat_audio_route(
|
||||
// 清理上传的临时音频
|
||||
let _ = std::fs::remove_file(&audio_path_clone);
|
||||
|
||||
let reset_after = if resp.error.is_none() {
|
||||
std::time::Duration::from_millis(
|
||||
((resp.reply_text.len() as u64) * 200).max(2000),
|
||||
)
|
||||
} else {
|
||||
std::time::Duration::from_millis(0)
|
||||
};
|
||||
let reset_state = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
if !reset_after.is_zero() {
|
||||
tokio::time::sleep(reset_after).await;
|
||||
}
|
||||
reset_state.set_live2d_talking(false);
|
||||
});
|
||||
|
||||
// 设备端字幕：把回复文字推给 live2d 插件做屏幕 Overlay
|
||||
if resp.error.is_none() && !resp.reply_text.is_empty() {
|
||||
state.send_message(
|
||||
|
||||
Reference in New Issue
Block a user