feat(Live2D): 设备端 Firefox kiosk 全屏渲染 Live2D 角色
Cubism Core 闭源且无 aarch64 Linux native 库，无法纯 Rust 原生渲染。 设备有 X11+kwin+Firefox 140+OpenGL，用 Firefox kiosk 模式全屏渲染 Live2D 模型（复用 Web 端 Cubism SDK），是 aarch64 上唯一可行路径。 1. src/plugins/http/live2d_display.html — 设备端专用全屏 Live2D 页面 - 纯 Canvas 渲染，无 UI 控件 - 轮询 /api/live2d/talking 驱动嘴部动效 - 自动从 /api/config 读取 live2d_model 路径 2. src/plugins/http/mod.rs — HttpState 加 live2d_talking AtomicBool - set_live2d_talking/getter 方法 3. src/plugins/http/routes.rs - 新增 GET /live2d-display.html 路由 - 新增 GET /api/live2d/talking 路由（设备端轮询） - chat_text_route 对话开始时 set_live2d_talking(true) - 回复后按字数估算时长延迟 set_live2d_talking(false) 4. src/plugins/video/mod.rs — Live2D 模式管理 Firefox kiosk 进程 - ConfigReloaded render_type=live2d：停止视频，启动 firefox --kiosk - ConfigReloaded render_type=video：杀掉 Firefox，恢复视频播放 - stop() 清理 Firefox 子进程
This commit is contained in:
72
src/plugins/http/live2d_display.html
Normal file
72
src/plugins/http/live2d_display.html
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Live2D Display</title>
|
||||||
|
<style>
|
||||||
|
*{margin:0;padding:0;box-sizing:border-box}
|
||||||
|
html,body{width:100%;height:100%;background:#000;overflow:hidden}
|
||||||
|
#canvas{display:block;width:100%;height:100%}
|
||||||
|
#status{position:fixed;top:8px;left:8px;color:#888;font:12px monospace;z-index:10}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<canvas id="canvas"></canvas>
|
||||||
|
<div id="status">加载中...</div>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/pixi.js@6.5.10/dist/browser/pixi.min.js"></script>
|
||||||
|
<script src="https://cubism.live2d.com/sdk-web/cubismcore/live2dcubismcore.min.js"></script>
|
||||||
|
<script src="https://cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js"></script>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/pixi-live2d-display/dist/index.min.js"></script>
|
||||||
|
<script>
|
||||||
|
window.PIXI=PIXI;
|
||||||
|
var app,model,talking=false;
|
||||||
|
var statusEl=document.getElementById('status');
|
||||||
|
var canvas=document.getElementById('canvas');
|
||||||
|
function setStatus(s){statusEl.textContent=s}
|
||||||
|
async function init(){
|
||||||
|
try{
|
||||||
|
var res=await fetch('/api/config');
|
||||||
|
var cfg=await res.json();
|
||||||
|
var char=cfg.character||{};
|
||||||
|
var modelPath=char.live2d_model;
|
||||||
|
if(!modelPath){setStatus('未配置 live2d_model');return}
|
||||||
|
var w=window.innerWidth,h=window.innerHeight;
|
||||||
|
app=new PIXI.Application({view:canvas,backgroundAlpha:0,width:w,height:h,resolution:window.devicePixelRatio||1,autoDensity:true});
|
||||||
|
var modelUrl='/live2d/'+modelPath.replace(/^\//,'').split('/').map(encodeURIComponent).join('/');
|
||||||
|
setStatus('加载模型: '+modelPath);
|
||||||
|
model=await PIXI.live2d.Live2DModel.from(modelUrl);
|
||||||
|
app.stage.addChild(model);
|
||||||
|
model.anchor.set(0.5,0.5);
|
||||||
|
model.x=app.screen.width/2;
|
||||||
|
model.y=app.screen.height/2;
|
||||||
|
var scale=Math.min(app.screen.width/model.width,app.screen.height/model.height)*0.9;
|
||||||
|
model.scale.set(scale);
|
||||||
|
setStatus('');
|
||||||
|
app.ticker.add(function(){
|
||||||
|
if(!model||!model.internalModel||!model.internalModel.coreModel)return;
|
||||||
|
if(talking){var v=Math.abs(Math.sin(Date.now()/120))*0.8+0.2;model.internalModel.coreModel.setParameterValueById('ParamMouthOpenY',v)}
|
||||||
|
else{model.internalModel.coreModel.setParameterValueById('ParamMouthOpenY',0)}
|
||||||
|
var breath=Math.sin(Date.now()/2000)*0.05;
|
||||||
|
model.internalModel.coreModel.setParameterValueById('ParamBreath',breath);
|
||||||
|
});
|
||||||
|
// 轮询对话状态
|
||||||
|
setInterval(pollTalking,500);
|
||||||
|
}catch(e){setStatus('错误: '+e.message)}
|
||||||
|
}
|
||||||
|
var lastTalk=false;
|
||||||
|
async function pollTalking(){
|
||||||
|
try{
|
||||||
|
var res=await fetch('/api/live2d/talking');
|
||||||
|
if(res.ok){var d=await res.json();talking=!!d.talking}
|
||||||
|
}catch(e){}
|
||||||
|
}
|
||||||
|
window.addEventListener('resize',function(){
|
||||||
|
if(!app)return;
|
||||||
|
var w=window.innerWidth,h=window.innerHeight;
|
||||||
|
app.renderer.resize(w,h);
|
||||||
|
if(model){model.x=app.screen.width/2;model.y=app.screen.height/2;var s=Math.min(app.screen.width/model.width,app.screen.height/model.height)*0.9;model.scale.set(s)}
|
||||||
|
});
|
||||||
|
init();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -52,6 +52,8 @@ pub(crate) struct HttpState {
|
|||||||
ai_pipeline: Mutex<Option<std::sync::Arc<crate::plugins::ai::ChatPipeline>>>,
|
ai_pipeline: Mutex<Option<std::sync::Arc<crate::plugins::ai::ChatPipeline>>>,
|
||||||
/// AI 模型管理器共享句柄（HTTP 模型管理 API 调用）
|
/// AI 模型管理器共享句柄（HTTP 模型管理 API 调用）
|
||||||
ai_models: Mutex<Option<std::sync::Arc<std::sync::Mutex<crate::plugins::ai::model_manager::ModelManager>>>>,
|
ai_models: Mutex<Option<std::sync::Arc<std::sync::Mutex<crate::plugins::ai::model_manager::ModelManager>>>>,
|
||||||
|
/// Live2D 说话状态（设备端 Firefox 轮询，驱动嘴部动效）
|
||||||
|
live2d_talking: AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HttpState {
|
impl HttpState {
|
||||||
@@ -81,6 +83,7 @@ impl HttpState {
|
|||||||
plugin_states: Mutex::new(Vec::new()),
|
plugin_states: Mutex::new(Vec::new()),
|
||||||
ai_pipeline: Mutex::new(None),
|
ai_pipeline: Mutex::new(None),
|
||||||
ai_models: Mutex::new(None),
|
ai_models: Mutex::new(None),
|
||||||
|
live2d_talking: AtomicBool::new(false),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,6 +116,16 @@ impl HttpState {
|
|||||||
self.ai_models.lock().ok().and_then(|slot| slot.clone())
|
self.ai_models.lock().ok().and_then(|slot| slot.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 设置 Live2D 说话状态（对话开始/结束时调用）
|
||||||
|
pub(crate) fn set_live2d_talking(&self, talking: bool) {
|
||||||
|
self.live2d_talking.store(talking, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取 Live2D 说话状态（设备端轮询接口调用）
|
||||||
|
pub(crate) fn live2d_talking(&self) -> bool {
|
||||||
|
self.live2d_talking.load(Ordering::SeqCst)
|
||||||
|
}
|
||||||
|
|
||||||
fn publish_wifi_result(&self, payload: String) {
|
fn publish_wifi_result(&self, payload: String) {
|
||||||
if let Ok(mut state) = self.wifi_response.lock() {
|
if let Ok(mut state) = self.wifi_response.lock() {
|
||||||
state.version += 1;
|
state.version += 1;
|
||||||
|
|||||||
@@ -149,6 +149,7 @@ pub(crate) fn build_routes(
|
|||||||
let ai_api = chat_text_route(Arc::clone(&state))
|
let ai_api = chat_text_route(Arc::clone(&state))
|
||||||
.or(chat_audio_route(Arc::clone(&state)))
|
.or(chat_audio_route(Arc::clone(&state)))
|
||||||
.or(chat_audio_file_route(Arc::clone(&state)))
|
.or(chat_audio_file_route(Arc::clone(&state)))
|
||||||
|
.or(live2d_talking_route(Arc::clone(&state)))
|
||||||
.or(models_list_route(Arc::clone(&state)))
|
.or(models_list_route(Arc::clone(&state)))
|
||||||
.or(model_download_route(Arc::clone(&state)))
|
.or(model_download_route(Arc::clone(&state)))
|
||||||
.or(model_switch_route(Arc::clone(&state)))
|
.or(model_switch_route(Arc::clone(&state)))
|
||||||
@@ -202,7 +203,11 @@ fn root_route() -> impl Filter<Extract = impl Reply, Error = warp::Rejection> +
|
|||||||
.into_response(),
|
.into_response(),
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
index.or(index_html).or(chat).or(chat_js)
|
let live2d_display = warp::path("live2d-display.html")
|
||||||
|
.and(warp::path::end())
|
||||||
|
.and(warp::get())
|
||||||
|
.map(|| warp::reply::html(include_str!("live2d_display.html")));
|
||||||
|
index.or(index_html).or(chat).or(chat_js).or(live2d_display)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ws_route(
|
fn ws_route(
|
||||||
@@ -2155,6 +2160,9 @@ fn chat_text_route(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// AI 管线是同步阻塞调用（ASR/LLM/TTS 子进程），用 spawn_blocking 避免阻塞 tokio reactor
|
// AI 管线是同步阻塞调用（ASR/LLM/TTS 子进程），用 spawn_blocking 避免阻塞 tokio reactor
|
||||||
|
// 设置 Live2D 说话状态（设备端 Firefox 轮询驱动嘴部动效）
|
||||||
|
let talking_state = Arc::clone(&state);
|
||||||
|
talking_state.set_live2d_talking(true);
|
||||||
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| ChatResponse {
|
.unwrap_or_else(|e| ChatResponse {
|
||||||
@@ -2164,6 +2172,21 @@ fn chat_text_route(
|
|||||||
reply_audio_path: None,
|
reply_audio_path: None,
|
||||||
error: Some(format!("AI 管线执行失败: {e}")),
|
error: Some(format!("AI 管线执行失败: {e}")),
|
||||||
});
|
});
|
||||||
|
// 回复音频时长估算后重置 talking（按字数估算）
|
||||||
|
let reset_after = if resp.error.is_none() {
|
||||||
|
std::time::Duration::from_millis(
|
||||||
|
((resp.reply_text.len() as u64) * 200).max(2000),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
std::time::Duration::from_millis(0)
|
||||||
|
};
|
||||||
|
let reset_state = Arc::clone(&state);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if !reset_after.is_zero() {
|
||||||
|
tokio::time::sleep(reset_after).await;
|
||||||
|
}
|
||||||
|
reset_state.set_live2d_talking(false);
|
||||||
|
});
|
||||||
|
|
||||||
if resp.error.is_some() {
|
if resp.error.is_some() {
|
||||||
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
|
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
|
||||||
@@ -2346,6 +2369,21 @@ fn chat_audio_file_route(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /api/live2d/talking — 返回当前 Live2D 说话状态（设备端 Firefox 轮询）
|
||||||
|
fn live2d_talking_route(
|
||||||
|
state: Arc<HttpState>,
|
||||||
|
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
|
||||||
|
warp::path!("api" / "live2d" / "talking")
|
||||||
|
.and(warp::get())
|
||||||
|
.and(with_state(state))
|
||||||
|
.and_then(|state: Arc<HttpState>| async move {
|
||||||
|
Ok::<_, Infallible>(json_response(
|
||||||
|
StatusCode::OK,
|
||||||
|
&serde_json::json!({ "talking": state.live2d_talking() }),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ── AI 模型管理 API (M2.1) ──
|
// ── AI 模型管理 API (M2.1) ──
|
||||||
|
|
||||||
/// GET /api/models — 列出所有模型
|
/// GET /api/models — 列出所有模型
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ pub struct VideoPlugin {
|
|||||||
ctx: Option<PluginContext>,
|
ctx: Option<PluginContext>,
|
||||||
processor: Option<Arc<Mutex<VideoProcessor>>>,
|
processor: Option<Arc<Mutex<VideoProcessor>>>,
|
||||||
worker: Option<JoinHandle<()>>,
|
worker: Option<JoinHandle<()>>,
|
||||||
|
/// Live2D 模式下运行的 Firefox kiosk 子进程
|
||||||
|
live2d_child: Option<std::process::Child>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VideoPlugin {
|
impl VideoPlugin {
|
||||||
@@ -26,6 +28,7 @@ impl VideoPlugin {
|
|||||||
ctx: None,
|
ctx: None,
|
||||||
processor: None,
|
processor: None,
|
||||||
worker: None,
|
worker: None,
|
||||||
|
live2d_child: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,6 +198,53 @@ impl Plugin for VideoPlugin {
|
|||||||
self.publish_status();
|
self.publish_status();
|
||||||
}
|
}
|
||||||
Message::ConfigReloaded(config) => {
|
Message::ConfigReloaded(config) => {
|
||||||
|
// Live2D 模式：停止视频播放，启动 Firefox kiosk 全屏渲染 Live2D 模型
|
||||||
|
if config.character.render_type == crate::core::config::RenderType::Live2d {
|
||||||
|
// 停止视频播放
|
||||||
|
if let Some(old) = self.processor.take() {
|
||||||
|
if let Ok(mut old) = old.lock() {
|
||||||
|
let _ = old.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(handle) = self.worker.take() {
|
||||||
|
let _ = handle.join();
|
||||||
|
}
|
||||||
|
// 杀掉旧的 Firefox（如有）
|
||||||
|
if let Some(mut child) = self.live2d_child.take() {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
}
|
||||||
|
// 启动 Firefox kiosk 模式全屏显示 Live2D
|
||||||
|
let port = config.remote_control.port;
|
||||||
|
let url = format!("http://localhost:{port}/live2d-display.html");
|
||||||
|
match std::process::Command::new("firefox")
|
||||||
|
.arg("--kiosk")
|
||||||
|
.arg("--width").arg(config.display.render_width.to_string())
|
||||||
|
.arg("--height").arg(config.display.render_height.to_string())
|
||||||
|
.arg(&url)
|
||||||
|
.env("DISPLAY", ":0")
|
||||||
|
.env("MOZ_DISABLE_AUTOPLAY", "1")
|
||||||
|
.spawn()
|
||||||
|
{
|
||||||
|
Ok(child) => {
|
||||||
|
self.live2d_child = Some(child);
|
||||||
|
println!("[VideoPlugin] Live2D 模式：Firefox kiosk 已启动 ({url})");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("[VideoPlugin] 启动 Firefox 失败: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.publish_status();
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 切回视频模式：杀掉 Firefox kiosk
|
||||||
|
if let Some(mut child) = self.live2d_child.take() {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
println!("[VideoPlugin] 切回视频模式：Firefox kiosk 已关闭");
|
||||||
|
}
|
||||||
|
|
||||||
let processor = Arc::new(Mutex::new(VideoProcessor::new(config)?));
|
let processor = Arc::new(Mutex::new(VideoProcessor::new(config)?));
|
||||||
if let Some(old) = self.processor.replace(Arc::clone(&processor)) {
|
if let Some(old) = self.processor.replace(Arc::clone(&processor)) {
|
||||||
if let Ok(mut old) = old.lock() {
|
if let Ok(mut old) = old.lock() {
|
||||||
@@ -240,6 +290,12 @@ impl Plugin for VideoPlugin {
|
|||||||
let _ = handle.join();
|
let _ = handle.join();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 关闭 Live2D Firefox kiosk
|
||||||
|
if let Some(mut child) = self.live2d_child.take() {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
}
|
||||||
|
|
||||||
self.publish_status();
|
self.publish_status();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user