feat(Live2D): 设备端 Firefox kiosk 全屏渲染 Live2D 角色
Cubism Core 闭源且无 aarch64 Linux native 库，无法纯 Rust 原生渲染。 设备有 X11+kwin+Firefox 140+OpenGL，用 Firefox kiosk 模式全屏渲染 Live2D 模型（复用 Web 端 Cubism SDK），是 aarch64 上唯一可行路径。 1. src/plugins/http/live2d_display.html — 设备端专用全屏 Live2D 页面 - 纯 Canvas 渲染，无 UI 控件 - 轮询 /api/live2d/talking 驱动嘴部动效 - 自动从 /api/config 读取 live2d_model 路径 2. src/plugins/http/mod.rs — HttpState 加 live2d_talking AtomicBool - set_live2d_talking/getter 方法 3. src/plugins/http/routes.rs - 新增 GET /live2d-display.html 路由 - 新增 GET /api/live2d/talking 路由（设备端轮询） - chat_text_route 对话开始时 set_live2d_talking(true) - 回复后按字数估算时长延迟 set_live2d_talking(false) 4. src/plugins/video/mod.rs — Live2D 模式管理 Firefox kiosk 进程 - ConfigReloaded render_type=live2d：停止视频，启动 firefox --kiosk - ConfigReloaded render_type=video：杀掉 Firefox，恢复视频播放 - stop() 清理 Firefox 子进程
This commit is contained in:
72
src/plugins/http/live2d_display.html
Normal file
72
src/plugins/http/live2d_display.html
Normal file
@@ -0,0 +1,72 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Live2D Display</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
html,body{width:100%;height:100%;background:#000;overflow:hidden}
|
||||
#canvas{display:block;width:100%;height:100%}
|
||||
#status{position:fixed;top:8px;left:8px;color:#888;font:12px monospace;z-index:10}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="canvas"></canvas>
|
||||
<div id="status">加载中...</div>
|
||||
<script src="https://cdn.jsdelivr.net/npm/pixi.js@6.5.10/dist/browser/pixi.min.js"></script>
|
||||
<script src="https://cubism.live2d.com/sdk-web/cubismcore/live2dcubismcore.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/pixi-live2d-display/dist/index.min.js"></script>
|
||||
<script>
|
||||
window.PIXI=PIXI;
|
||||
var app,model,talking=false;
|
||||
var statusEl=document.getElementById('status');
|
||||
var canvas=document.getElementById('canvas');
|
||||
function setStatus(s){statusEl.textContent=s}
|
||||
async function init(){
|
||||
try{
|
||||
var res=await fetch('/api/config');
|
||||
var cfg=await res.json();
|
||||
var char=cfg.character||{};
|
||||
var modelPath=char.live2d_model;
|
||||
if(!modelPath){setStatus('未配置 live2d_model');return}
|
||||
var w=window.innerWidth,h=window.innerHeight;
|
||||
app=new PIXI.Application({view:canvas,backgroundAlpha:0,width:w,height:h,resolution:window.devicePixelRatio||1,autoDensity:true});
|
||||
var modelUrl='/live2d/'+modelPath.replace(/^\//,'').split('/').map(encodeURIComponent).join('/');
|
||||
setStatus('加载模型: '+modelPath);
|
||||
model=await PIXI.live2d.Live2DModel.from(modelUrl);
|
||||
app.stage.addChild(model);
|
||||
model.anchor.set(0.5,0.5);
|
||||
model.x=app.screen.width/2;
|
||||
model.y=app.screen.height/2;
|
||||
var scale=Math.min(app.screen.width/model.width,app.screen.height/model.height)*0.9;
|
||||
model.scale.set(scale);
|
||||
setStatus('');
|
||||
app.ticker.add(function(){
|
||||
if(!model||!model.internalModel||!model.internalModel.coreModel)return;
|
||||
if(talking){var v=Math.abs(Math.sin(Date.now()/120))*0.8+0.2;model.internalModel.coreModel.setParameterValueById('ParamMouthOpenY',v)}
|
||||
else{model.internalModel.coreModel.setParameterValueById('ParamMouthOpenY',0)}
|
||||
var breath=Math.sin(Date.now()/2000)*0.05;
|
||||
model.internalModel.coreModel.setParameterValueById('ParamBreath',breath);
|
||||
});
|
||||
// 轮询对话状态
|
||||
setInterval(pollTalking,500);
|
||||
}catch(e){setStatus('错误: '+e.message)}
|
||||
}
|
||||
var lastTalk=false;
|
||||
async function pollTalking(){
|
||||
try{
|
||||
var res=await fetch('/api/live2d/talking');
|
||||
if(res.ok){var d=await res.json();talking=!!d.talking}
|
||||
}catch(e){}
|
||||
}
|
||||
window.addEventListener('resize',function(){
|
||||
if(!app)return;
|
||||
var w=window.innerWidth,h=window.innerHeight;
|
||||
app.renderer.resize(w,h);
|
||||
if(model){model.x=app.screen.width/2;model.y=app.screen.height/2;var s=Math.min(app.screen.width/model.width,app.screen.height/model.height)*0.9;model.scale.set(s)}
|
||||
});
|
||||
init();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -52,6 +52,8 @@ pub(crate) struct HttpState {
|
||||
ai_pipeline: Mutex<Option<std::sync::Arc<crate::plugins::ai::ChatPipeline>>>,
|
||||
/// AI 模型管理器共享句柄（HTTP 模型管理 API 调用）
|
||||
ai_models: Mutex<Option<std::sync::Arc<std::sync::Mutex<crate::plugins::ai::model_manager::ModelManager>>>>,
|
||||
/// Live2D 说话状态（设备端 Firefox 轮询，驱动嘴部动效）
|
||||
live2d_talking: AtomicBool,
|
||||
}
|
||||
|
||||
impl HttpState {
|
||||
@@ -81,6 +83,7 @@ impl HttpState {
|
||||
plugin_states: Mutex::new(Vec::new()),
|
||||
ai_pipeline: Mutex::new(None),
|
||||
ai_models: Mutex::new(None),
|
||||
live2d_talking: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,6 +116,16 @@ impl HttpState {
|
||||
self.ai_models.lock().ok().and_then(|slot| slot.clone())
|
||||
}
|
||||
|
||||
/// 设置 Live2D 说话状态（对话开始/结束时调用）
|
||||
pub(crate) fn set_live2d_talking(&self, talking: bool) {
|
||||
self.live2d_talking.store(talking, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// 获取 Live2D 说话状态（设备端轮询接口调用）
|
||||
pub(crate) fn live2d_talking(&self) -> bool {
|
||||
self.live2d_talking.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn publish_wifi_result(&self, payload: String) {
|
||||
if let Ok(mut state) = self.wifi_response.lock() {
|
||||
state.version += 1;
|
||||
|
||||
@@ -149,6 +149,7 @@ pub(crate) fn build_routes(
|
||||
let ai_api = chat_text_route(Arc::clone(&state))
|
||||
.or(chat_audio_route(Arc::clone(&state)))
|
||||
.or(chat_audio_file_route(Arc::clone(&state)))
|
||||
.or(live2d_talking_route(Arc::clone(&state)))
|
||||
.or(models_list_route(Arc::clone(&state)))
|
||||
.or(model_download_route(Arc::clone(&state)))
|
||||
.or(model_switch_route(Arc::clone(&state)))
|
||||
@@ -202,7 +203,11 @@ fn root_route() -> impl Filter<Extract = impl Reply, Error = warp::Rejection> +
|
||||
.into_response(),
|
||||
}
|
||||
});
|
||||
index.or(index_html).or(chat).or(chat_js)
|
||||
let live2d_display = warp::path("live2d-display.html")
|
||||
.and(warp::path::end())
|
||||
.and(warp::get())
|
||||
.map(|| warp::reply::html(include_str!("live2d_display.html")));
|
||||
index.or(index_html).or(chat).or(chat_js).or(live2d_display)
|
||||
}
|
||||
|
||||
fn ws_route(
|
||||
@@ -2155,6 +2160,9 @@ fn chat_text_route(
|
||||
};
|
||||
|
||||
// AI 管线是同步阻塞调用（ASR/LLM/TTS 子进程），用 spawn_blocking 避免阻塞 tokio reactor
|
||||
// 设置 Live2D 说话状态（设备端 Firefox 轮询驱动嘴部动效）
|
||||
let talking_state = Arc::clone(&state);
|
||||
talking_state.set_live2d_talking(true);
|
||||
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
||||
.await
|
||||
.unwrap_or_else(|e| ChatResponse {
|
||||
@@ -2164,6 +2172,21 @@ fn chat_text_route(
|
||||
reply_audio_path: None,
|
||||
error: Some(format!("AI 管线执行失败: {e}")),
|
||||
});
|
||||
// 回复音频时长估算后重置 talking（按字数估算）
|
||||
let reset_after = if resp.error.is_none() {
|
||||
std::time::Duration::from_millis(
|
||||
((resp.reply_text.len() as u64) * 200).max(2000),
|
||||
)
|
||||
} else {
|
||||
std::time::Duration::from_millis(0)
|
||||
};
|
||||
let reset_state = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
if !reset_after.is_zero() {
|
||||
tokio::time::sleep(reset_after).await;
|
||||
}
|
||||
reset_state.set_live2d_talking(false);
|
||||
});
|
||||
|
||||
if resp.error.is_some() {
|
||||
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
|
||||
@@ -2346,6 +2369,21 @@ fn chat_audio_file_route(
|
||||
)
|
||||
}
|
||||
|
||||
/// GET /api/live2d/talking — 返回当前 Live2D 说话状态（设备端 Firefox 轮询）
|
||||
fn live2d_talking_route(
|
||||
state: Arc<HttpState>,
|
||||
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
|
||||
warp::path!("api" / "live2d" / "talking")
|
||||
.and(warp::get())
|
||||
.and(with_state(state))
|
||||
.and_then(|state: Arc<HttpState>| async move {
|
||||
Ok::<_, Infallible>(json_response(
|
||||
StatusCode::OK,
|
||||
&serde_json::json!({ "talking": state.live2d_talking() }),
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ── AI 模型管理 API (M2.1) ──
|
||||
|
||||
/// GET /api/models — 列出所有模型
|
||||
|
||||
@@ -18,6 +18,8 @@ pub struct VideoPlugin {
|
||||
ctx: Option<PluginContext>,
|
||||
processor: Option<Arc<Mutex<VideoProcessor>>>,
|
||||
worker: Option<JoinHandle<()>>,
|
||||
/// Live2D 模式下运行的 Firefox kiosk 子进程
|
||||
live2d_child: Option<std::process::Child>,
|
||||
}
|
||||
|
||||
impl VideoPlugin {
|
||||
@@ -26,6 +28,7 @@ impl VideoPlugin {
|
||||
ctx: None,
|
||||
processor: None,
|
||||
worker: None,
|
||||
live2d_child: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,6 +198,53 @@ impl Plugin for VideoPlugin {
|
||||
self.publish_status();
|
||||
}
|
||||
Message::ConfigReloaded(config) => {
|
||||
// Live2D 模式：停止视频播放，启动 Firefox kiosk 全屏渲染 Live2D 模型
|
||||
if config.character.render_type == crate::core::config::RenderType::Live2d {
|
||||
// 停止视频播放
|
||||
if let Some(old) = self.processor.take() {
|
||||
if let Ok(mut old) = old.lock() {
|
||||
let _ = old.stop();
|
||||
}
|
||||
}
|
||||
if let Some(handle) = self.worker.take() {
|
||||
let _ = handle.join();
|
||||
}
|
||||
// 杀掉旧的 Firefox（如有）
|
||||
if let Some(mut child) = self.live2d_child.take() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
// 启动 Firefox kiosk 模式全屏显示 Live2D
|
||||
let port = config.remote_control.port;
|
||||
let url = format!("http://localhost:{port}/live2d-display.html");
|
||||
match std::process::Command::new("firefox")
|
||||
.arg("--kiosk")
|
||||
.arg("--width").arg(config.display.render_width.to_string())
|
||||
.arg("--height").arg(config.display.render_height.to_string())
|
||||
.arg(&url)
|
||||
.env("DISPLAY", ":0")
|
||||
.env("MOZ_DISABLE_AUTOPLAY", "1")
|
||||
.spawn()
|
||||
{
|
||||
Ok(child) => {
|
||||
self.live2d_child = Some(child);
|
||||
println!("[VideoPlugin] Live2D 模式：Firefox kiosk 已启动 ({url})");
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("[VideoPlugin] 启动 Firefox 失败: {e}");
|
||||
}
|
||||
}
|
||||
self.publish_status();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// 切回视频模式：杀掉 Firefox kiosk
|
||||
if let Some(mut child) = self.live2d_child.take() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
println!("[VideoPlugin] 切回视频模式：Firefox kiosk 已关闭");
|
||||
}
|
||||
|
||||
let processor = Arc::new(Mutex::new(VideoProcessor::new(config)?));
|
||||
if let Some(old) = self.processor.replace(Arc::clone(&processor)) {
|
||||
if let Ok(mut old) = old.lock() {
|
||||
@@ -240,6 +290,12 @@ impl Plugin for VideoPlugin {
|
||||
let _ = handle.join();
|
||||
}
|
||||
|
||||
// 关闭 Live2D Firefox kiosk
|
||||
if let Some(mut child) = self.live2d_child.take() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
|
||||
self.publish_status();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user