diff --git a/Cargo.lock b/Cargo.lock index e08ff56..ee1d889 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1370e9fc2a6ae53aea8b7a5110edbd08836ed87c88736dfabccade1c2b44bff4" + [[package]] name = "bitflags" version = "2.11.0" @@ -215,7 +221,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.11.0", "block2", "libc", "objc2", @@ -314,6 +320,28 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "freetype-rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1418e2a055fec8efe18c1a90a54b2cf5e649e583830dd4c71226c4e3bc920c6" +dependencies = [ + "bitflags 0.8.2", + "freetype-sys", + "libc", +] + +[[package]] +name = "freetype-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eccfb6d96cac99921f0c2142a91765f6c219868a2c45bdfe7d65a08775f18127" +dependencies = [ + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -695,12 +723,24 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "plain", "redox_syscall", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -802,7 +842,7 @@ version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -924,7 +964,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crc32fast", "fdeflate", "flate2", @@ -1015,7 +1055,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -1067,7 +1107,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -1226,6 +1266,7 @@ dependencies = [ "dbus", "dbus-crossroads", "flate2", + "freetype-rs", "futures-util", "image", "libloading", diff --git a/Cargo.toml b/Cargo.toml index 943c10a..2461eda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,3 +40,5 @@ semver = "1" # Live2D 渲染 image = { version = "0.25", default-features = false, features = ["png"] } +# 设备端字幕（中文光栅化，调用系统 FreeType 高级封装） +freetype-rs = "0.13" diff --git a/src/core/message.rs b/src/core/message.rs index 35d2eb0..6e469b4 100644 --- a/src/core/message.rs +++ b/src/core/message.rs @@ -69,6 +69,10 @@ pub enum Message { /// AI 模型状态变更广播（加载/切换/下载进度） AiModelEvent(AiModelEvent), + // ── 设备端字幕 (M2.1) ── + /// 设备端 Live2D 字幕显示（AI 回复文字 Overlay）；HTTP 插件发出，live2d 插件消费 + Subtitle { text: String }, + // ── 扩展（未来插件用） ── Custom { kind: String, diff --git a/src/plugins/http/mod.rs b/src/plugins/http/mod.rs index 3f3cbf8..7c6a2bd 100644 --- a/src/plugins/http/mod.rs +++ b/src/plugins/http/mod.rs @@ -5,11 +5,12 @@ mod routes; use crate::core::config::AppConfig; -use crate::core::message::{Envelope, Message}; +use crate::core::message::{Destination, Envelope, Message}; use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo}; use anyhow::{Context, Result}; use serde::Serialize; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; use std::sync::{Arc, Condvar, Mutex}; use std::thread::JoinHandle; use tokio::sync::broadcast; @@ -54,10 +55,12 @@ pub(crate) struct HttpState { ai_models: Mutex>>>, /// Live2D 说话状态（设备端 Firefox 轮询，驱动嘴部动效） live2d_talking: AtomicBool, + /// 向其他插件（如 live2d）转发消息的出口（init 时由 PluginContext.tx 注入） + outbox: mpsc::Sender, } impl HttpState { - fn new(config: Arc) -> Self { + fn new(config: Arc, tx: mpsc::Sender) -> Self { let (ws_events, _) = broadcast::channel(32); let player_status = crate::core::message::PlayerStatusData { running: false, @@ -84,6 +87,18 @@ impl HttpState { ai_pipeline: Mutex::new(None), ai_models: Mutex::new(None), live2d_talking: AtomicBool::new(false), + outbox: tx, + } + } + + /// 向指定插件发送消息（用于把 AI 回复文字推给 live2d 做设备端字幕）。 + pub(crate) fn send_message(&self, to: Destination, msg: Message) { + if let Err(e) = self.outbox.send(Envelope { + from: "http".to_string(), + to, + message: msg, + }) { + eprintln!("[HttpPlugin] 发送消息失败: {e}"); } } @@ -312,7 +327,10 @@ impl Plugin for HttpPlugin { } fn init(&mut self, ctx: PluginContext) -> Result<()> { - let state = Arc::new(HttpState::new(Arc::clone(&ctx.config))); + let state = Arc::new(HttpState::new( + Arc::clone(&ctx.config), + ctx.tx.clone(), + )); // 注册 AI 句柄（如果 main.rs 已设置） if let Some(pipeline) = self.ai_pipeline.take() { state.register_ai_pipeline(pipeline); diff --git a/src/plugins/http/routes.rs b/src/plugins/http/routes.rs index 20f4503..27b55fd 100644 --- a/src/plugins/http/routes.rs +++ b/src/plugins/http/routes.rs @@ -2188,6 +2188,16 @@ fn chat_text_route( reset_state.set_live2d_talking(false); }); + // 设备端字幕：把回复文字推给 live2d 插件做屏幕 Overlay + if resp.error.is_none() && !resp.reply_text.is_empty() { + state.send_message( + Destination::Plugin("live2d".to_string()), + Message::Subtitle { + text: resp.reply_text.clone(), + }, + ); + } + if resp.error.is_some() { Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp)) } else { @@ -2274,6 +2284,16 @@ fn chat_audio_route( // 清理上传的临时音频 let _ = std::fs::remove_file(&audio_path_clone); + // 设备端字幕：把回复文字推给 live2d 插件做屏幕 Overlay + if resp.error.is_none() && !resp.reply_text.is_empty() { + state.send_message( + Destination::Plugin("live2d".to_string()), + Message::Subtitle { + text: resp.reply_text.clone(), + }, + ); + } + if resp.error.is_some() { Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp)) } else { diff --git a/src/plugins/live2d/gl.rs b/src/plugins/live2d/gl.rs index 9f30ff6..db763e7 100644 --- a/src/plugins/live2d/gl.rs +++ b/src/plugins/live2d/gl.rs @@ -205,6 +205,29 @@ void main() { } "#; +/// 文字 overlay 着色器（像素坐标 -> 裁剪空间，采样字形纹理，按 u_color 着色） +pub const TEXT_VERT_SHADER_SRC: &str = r#"#version 100 +attribute vec2 a_position; +attribute vec2 a_texCoord; +uniform mat4 u_matrix; +varying vec2 v_uv; +void main() { + gl_Position = u_matrix * vec4(a_position, 0.0, 1.0); + v_uv = a_texCoord; +} +"#; + +pub const TEXT_FRAG_SHADER_SRC: &str = r#"#version 100 +precision mediump float; +varying vec2 v_uv; +uniform sampler2D s_texture0; +uniform vec4 u_color; +void main() { + float a = texture2D(s_texture0, v_uv).a; + gl_FragColor = vec4(u_color.rgb, a * u_color.a); +} +"#; + /// 遮罩（mask）着色器：仅把 mask drawable 的不透明区域写入 stencil， /// 用于裁切被遮罩的 drawable（如夹在身体里的后臂、被头发遮挡的脸）。 pub const MASK_VERT_SHADER_SRC: &str = r#"#version 100 @@ -255,6 +278,12 @@ pub struct Gl { config_size: egl::EGLint, num_config: *mut egl::EGLint, ) -> egl::EGLBoolean, + pub egl_get_config_attrib: unsafe extern "C" fn( + dpy: egl::EGLDisplay, + config: egl::EGLConfig, + attribute: egl::EGLint, + value: *mut egl::EGLint, + ) -> egl::EGLBoolean, pub egl_create_window_surface: unsafe extern "C" fn( dpy: egl::EGLDisplay, config: egl::EGLConfig, @@ -433,6 +462,7 @@ impl Gl { egl_get_display: *lib_egl.get(b"eglGetDisplay\0")?, egl_initialize: *lib_egl.get(b"eglInitialize\0")?, egl_choose_config: *lib_egl.get(b"eglChooseConfig\0")?, + egl_get_config_attrib: *lib_egl.get(b"eglGetConfigAttrib\0")?, egl_create_window_surface: *lib_egl.get(b"eglCreateWindowSurface\0")?, egl_create_context: *lib_egl.get(b"eglCreateContext\0")?, egl_make_current: *lib_egl.get(b"eglMakeCurrent\0")?, @@ -530,6 +560,13 @@ pub struct RenderContext { pub mask_uniform_texture: GLint, pub mask_uniform_multiply_color: GLint, pub mask_uniform_screen_color: GLint, + // 文字 overlay 程序 + pub text_program: GLuint, + pub text_uniform_matrix: GLint, + pub text_uniform_color: GLint, + pub text_uniform_texture: GLint, + pub text_attrib_position: GLint, + pub text_attrib_tex_coord: GLint, pub width: i32, pub height: i32, } @@ -714,6 +751,18 @@ impl RenderContext { num_config ); + // 真实查询 chosen config 的 depth/stencil 位数，确认 stencil 缓冲真正分配。 + let mut stencil_size: egl::EGLint = -1; + let mut depth_size: egl::EGLint = -1; + unsafe { + (gl.egl_get_config_attrib)(display, config, egl::EGL_STENCIL_SIZE, &mut stencil_size); + (gl.egl_get_config_attrib)(display, config, egl::EGL_DEPTH_SIZE, &mut depth_size); + } + println!( + "[Live2D] EGL 实际分配: depth={} stencil={} (stencil<=0 表示遮罩不可用)", + depth_size, stencil_size + ); + let surface = unsafe { (gl.egl_create_window_surface)(display, config, x_window as egl::NativeWindowType, std::ptr::null()) }; @@ -846,6 +895,44 @@ impl RenderContext { unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) } }; + // 文字 overlay 程序 + let text_vert = compile_shader(&gl, gl::GL_VERTEX_SHADER, TEXT_VERT_SHADER_SRC)?; + let text_frag = compile_shader(&gl, gl::GL_FRAGMENT_SHADER, TEXT_FRAG_SHADER_SRC)?; + let text_program = unsafe { + let p = (gl.gl_create_program)(); + (gl.gl_attach_shader)(p, text_vert); + (gl.gl_attach_shader)(p, text_frag); + (gl.gl_link_program)(p); + let mut status = 0; + (gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status); + if status == 0 { + let mut log: Vec = vec![0u8; 1024]; + (gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char); + return Err(anyhow!("text shader link failed: {}", String::from_utf8_lossy(&log))); + } + p + }; + let text_uniform_matrix = { + let c = std::ffi::CString::new("u_matrix")?; + unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) } + }; + let text_uniform_color = { + let c = std::ffi::CString::new("u_color")?; + unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) } + }; + let text_uniform_texture = { + let c = std::ffi::CString::new("s_texture0")?; + unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) } + }; + let text_attrib_position = { + let c = std::ffi::CString::new("a_position")?; + unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) } + }; + let text_attrib_tex_coord = { + let c = std::ffi::CString::new("a_texCoord")?; + unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) } + }; + println!( "[Live2D] GL 程序就绪: program={} pos={} tex={} mat={} tex_u={} base={} mult={} scr={}", program, attrib_position, attrib_tex_coord, uniform_matrix, @@ -882,6 +969,12 @@ impl RenderContext { mask_uniform_texture, mask_uniform_multiply_color, mask_uniform_screen_color, + text_program, + text_uniform_matrix, + text_uniform_color, + text_uniform_texture, + text_attrib_position, + text_attrib_tex_coord, width: win_w, height: win_h, }) diff --git a/src/plugins/live2d/mod.rs b/src/plugins/live2d/mod.rs index 86182f7..c474191 100644 --- a/src/plugins/live2d/mod.rs +++ b/src/plugins/live2d/mod.rs @@ -16,6 +16,7 @@ pub mod gl; pub mod model; pub mod pose; pub mod renderer; +pub mod text; mod plugin; pub use plugin::Live2DPlugin; diff --git a/src/plugins/live2d/plugin.rs b/src/plugins/live2d/plugin.rs index 580c8ab..e9a1649 100644 --- a/src/plugins/live2d/plugin.rs +++ b/src/plugins/live2d/plugin.rs @@ -9,10 +9,11 @@ use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo}; use anyhow::{Context, Result}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; +use std::time::{Duration, Instant}; -use super::renderer::Live2DRenderer; +use super::renderer::{Live2DRenderer, Subtitle, SubtitleState}; pub struct Live2DPlugin { ctx: Option, @@ -21,6 +22,8 @@ pub struct Live2DPlugin { worker: Option>, /// 渲染停止信号 stop_flag: Arc, + /// 字幕共享状态（http 插件写入，渲染线程读取） + subtitle: SubtitleState, } impl Live2DPlugin { @@ -30,6 +33,7 @@ impl Live2DPlugin { renderer: None, worker: None, stop_flag: Arc::new(AtomicBool::new(false)), + subtitle: Arc::new(Mutex::new(None)), } } } @@ -97,6 +101,16 @@ impl Plugin for Live2DPlugin { Message::Shutdown => { self.stop_live2d(); } + Message::Subtitle { text } => { + // 设备端字幕：写入共享状态，渲染线程每帧读取，超时自动消失 + let ttl = Duration::from_millis( + (text.chars().count().max(1) as u64) + .saturating_mul(220) + .saturating_add(2500), + ); + *self.subtitle.lock().unwrap() = + Some(Subtitle { text, expires_at: Instant::now() + ttl }); + } _ => {} } Ok(()) @@ -139,7 +153,14 @@ impl Live2DPlugin { .clone(); let renderer = Arc::new(std::sync::Mutex::new( - Live2DRenderer::new(&core_so, &live2d_base, &model_rel, width, height)?, + Live2DRenderer::new( + &core_so, + &live2d_base, + &model_rel, + width, + height, + Arc::clone(&self.subtitle), + )?, )); self.renderer = Some(Arc::clone(&renderer)); diff --git a/src/plugins/live2d/renderer.rs b/src/plugins/live2d/renderer.rs index e93939d..26a9517 100644 --- a/src/plugins/live2d/renderer.rs +++ b/src/plugins/live2d/renderer.rs @@ -7,11 +7,26 @@ use super::gl::{self as gl_api, RenderContext}; use gl_api::gl as gl_const; use super::model::{CubismModel, DrawableSnapshot}; use super::pose::PoseState; +use super::text::TextRenderer; use anyhow::Result; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::{Duration, Instant}; +/// 前几帧打印被遮罩 drawable 的 mask 引用关系，便于排查遮罩错误（如后臂整只伸开）。 +static LIVE2D_MASK_DEBUG: AtomicU32 = AtomicU32::new(0); + +/// 设备端字幕：AI 回复文字 + 自动消失时间。 +pub struct Subtitle { + pub text: String, + pub expires_at: Instant, +} + +/// 字幕共享状态（http 插件写入，live2d 渲染线程读取）。 +pub type SubtitleState = Arc>>; + pub struct Live2DRenderer { pub core: Arc, pub model: CubismModel, @@ -23,6 +38,10 @@ pub struct Live2DRenderer { pub pose: Option, pub model_matrix: [f32; 16], pub running: bool, + /// 字幕共享状态（由 live2d 插件从 http 插件接收并传入） + pub subtitle: SubtitleState, + /// 中文字幕渲染器（无可用字体时为 None） + pub text: Option, } impl Live2DRenderer { @@ -32,6 +51,7 @@ impl Live2DRenderer { model3_json_rel: &str, width: i32, height: i32, + subtitle: SubtitleState, ) -> Result { let core = Arc::new(CubismCore::load(core_so_path)?); println!("[Live2D] Core version: {}", core.version_string()); @@ -68,6 +88,21 @@ impl Live2DRenderer { pose.reset(&core, &model); } + // 初始化中文字幕渲染器（找不到字体则降级为无字幕） + let text = match find_cjk_font() { + Some(p) => match TextRenderer::new(p.as_path(), 44) { + Ok(t) => Some(t), + Err(e) => { + eprintln!("[Live2D] 字幕渲染器初始化失败，字幕功能禁用: {e}"); + None + } + }, + None => { + eprintln!("[Live2D] 未找到中文字体，字幕功能禁用"); + None + } + }; + Ok(Self { core, model, @@ -79,6 +114,8 @@ impl Live2DRenderer { pose, model_matrix, running: false, + subtitle, + text, }) } @@ -98,6 +135,39 @@ impl Live2DRenderer { let snapshots = self.model.drawable_snapshots(&self.core); + // 前 1 帧打印全部 drawable 的概要 + 顶点包围盒，用于排查遮罩（[Live2D][maskdbg]） + let dbg = LIVE2D_MASK_DEBUG.fetch_add(1, Ordering::SeqCst); + if dbg < 1 { + eprintln!("[Live2D][maskdbg] total={}", snapshots.len()); + for s in &snapshots { + let (mut minx, mut miny, mut maxx, mut maxy) = + (f32::MAX, f32::MAX, f32::MIN, f32::MIN); + for v in &s.vertex_positions { + minx = minx.min(v[0]); + miny = miny.min(v[1]); + maxx = maxx.max(v[0]); + maxy = maxy.max(v[1]); + } + let inv = (s.constant_flags & super::core_ffi::constant_flags::IS_INVERTED_MASK) + != 0; + eprintln!( + "[Live2D][maskdbg] i={} ro={} op={:.2} fl={:#04x} masked={} masks={:?} inv={} bbox=({:.0},{:.0})-({:.0},{:.0}) tex={}", + s.index, + s.render_order, + s.opacity, + s.constant_flags, + !s.mask_indices.is_empty(), + s.mask_indices, + inv, + minx, + miny, + maxx, + maxy, + s.texture_index + ); + } + } + unsafe { let gl = &self.gl_ctx.gl; // 每帧校正 viewport 为窗口实际尺寸，避免窗口被 WM 调整大小后 @@ -135,6 +205,9 @@ impl Live2DRenderer { } } + // 设备端字幕 overlay（AI 回复文字） + self.draw_subtitle(); + self.model.reset_dynamic_flags(&self.core); unsafe { @@ -259,7 +332,7 @@ impl Live2DRenderer { (gl.gl_clear)(gl_const::GL_STENCIL_BUFFER_BIT); (gl.gl_color_mask)(0, 0, 0, 0); (gl.gl_stencil_func)(gl_const::GL_ALWAYS, 1, 0xFF); - (gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_REPLACE); + (gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_INCR); (gl.gl_use_program)(self.gl_ctx.mask_program); (gl.gl_uniform_matrix4fv)( @@ -384,6 +457,117 @@ impl Live2DRenderer { } } + /// 绘制设备端字幕：屏幕底部半透明背景框 + 白色中文（AI 回复文字）。 + fn draw_subtitle(&mut self) { + let (text, expired) = { + let guard = self.subtitle.lock().unwrap(); + match guard.as_ref() { + Some(s) => (s.text.clone(), s.expires_at <= Instant::now()), + None => return, + } + }; + if expired || text.trim().is_empty() { + return; + } + let text_renderer = match self.text.as_mut() { + Some(t) => t, + None => return, + }; + + let gl = &self.gl_ctx.gl; + // 复制 text program 的 uniform/attrib 位置到局部，避免与 &mut text_renderer 借位冲突 + let prog = self.gl_ctx.text_program; + let u_mat = self.gl_ctx.text_uniform_matrix; + let u_col = self.gl_ctx.text_uniform_color; + let u_tex = self.gl_ctx.text_uniform_texture; + let a_pos = self.gl_ctx.text_attrib_position; + let a_tco = self.gl_ctx.text_attrib_tex_coord; + + unsafe { + (gl.gl_enable)(gl_const::GL_BLEND); + (gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE_MINUS_SRC_ALPHA); + (gl.gl_use_program)(prog); + let w = self.gl_ctx.width as f32; + let h = self.gl_ctx.height as f32; + // 像素坐标（原点左上，y 向下）-> 裁剪空间 + let ortho: [f32; 16] = [ + 2.0 / w, 0.0, 0.0, 0.0, + 0.0, -2.0 / h, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, + -1.0, 1.0, 0.0, 1.0, + ]; + (gl.gl_uniform_matrix4fv)(u_mat, 1, gl_const::GL_FALSE, ortho.as_ptr()); + } + + let font_px = text_renderer.font_size() as f32; + let line_height = (font_px * 1.35) as i32; + let margin = 40i32; + let max_width = self.gl_ctx.width - margin * 2; + let padding = 18i32; + + // 按宽度换行（CJK 任意位置可断，遇 \n 强制断） + let mut lines: Vec = Vec::new(); + let mut cur = String::new(); + let mut cur_w = 0i32; + for ch in text.chars() { + if ch == '\n' { + lines.push(std::mem::take(&mut cur)); + cur_w = 0; + continue; + } + let aw = text_renderer.measure_advance(ch); + if cur_w + aw > max_width && !cur.is_empty() { + lines.push(std::mem::take(&mut cur)); + cur_w = 0; + } + cur.push(ch); + cur_w += aw; + } + if !cur.is_empty() { + lines.push(cur); + } + if lines.is_empty() { + return; + } + + let total_text_h = lines.len() as i32 * line_height; + let box_h = total_text_h + padding * 2; + let box_y = self.gl_ctx.height - margin - box_h; + let box_x = margin; + let box_w = self.gl_ctx.width - margin * 2; + + // 背景框 + let white = text_renderer.white_texture(gl); + draw_text_quad( + gl, white, box_x as f32, box_y as f32, box_w as f32, box_h as f32, [0.0, 0.0, 0.0, 0.55], + u_tex, u_col, a_pos, a_tco, + ); + + // 逐行绘制文字（居中） + for (li, line) in lines.iter().enumerate() { + let line_w: i32 = line.chars().map(|c| text_renderer.measure_advance(c)).sum(); + let start_x = ((self.gl_ctx.width - line_w) / 2) as f32; + let baseline_y = + (box_y + padding + li as i32 * line_height + (line_height as f32 * 0.78) as i32) as f32; + let mut pen_x = start_x; + for ch in line.chars() { + if let Some(g) = text_renderer.glyph(gl, ch) { + if g.tex != 0 && g.w > 0 && g.h > 0 { + let gx = pen_x + g.left as f32; + let gy = baseline_y - g.top as f32; + draw_text_quad( + gl, g.tex, gx, gy, g.w as f32, g.h as f32, [1.0, 1.0, 1.0, 1.0], + u_tex, u_col, a_pos, a_tco, + ); + } + pen_x += (g.advance >> 6) as f32; + } else { + pen_x += font_px * 0.5; + } + } + } + } + pub fn run_loop(&mut self) -> Result<()> { self.running = true; let frame_duration = Duration::from_millis(16); @@ -442,3 +626,83 @@ fn compute_model_matrix( 0.0, 0.0, 0.0, 1.0, ] } + +/// 用 text program 画一个带纹理的四边形（TRIANGLE_STRIP）。 +/// +/// `color` 为 RGBA；着色器输出 = u_color.rgb 配合字形纹理的 alpha，因此背景框可传 +/// 纯色 + alpha，字形可传白色 + alpha=1（字形纹理本身含 alpha 覆盖）。 +fn draw_text_quad( + gl: &gl_api::Gl, + tex: gl_const::GLuint, + x: f32, + y: f32, + w: f32, + h: f32, + color: [f32; 4], + uniform_texture: gl_const::GLint, + uniform_color: gl_const::GLint, + attrib_position: gl_const::GLint, + attrib_tex_coord: gl_const::GLint, +) { + unsafe { + (gl.gl_active_texture)(gl_const::GL_TEXTURE0); + (gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex); + (gl.gl_uniform1i)(uniform_texture, 0); + (gl.gl_uniform4f)(uniform_color, color[0], color[1], color[2], color[3]); + let verts: [f32; 16] = [ + x, y, 0.0, 0.0, + x + w, y, 1.0, 0.0, + x, y + h, 0.0, 1.0, + x + w, y + h, 1.0, 1.0, + ]; + let mut vbo = 0u32; + (gl.gl_gen_buffers)(1, &mut vbo); + (gl.gl_bind_buffer)(gl_const::GL_ARRAY_BUFFER, vbo); + (gl.gl_buffer_data)( + gl_const::GL_ARRAY_BUFFER, + (verts.len() * 4) as isize, + verts.as_ptr() as *const std::ffi::c_void, + gl_const::GL_STATIC_DRAW, + ); + let stride = 4 * 4; + if attrib_position >= 0 { + (gl.gl_enable_vertex_attrib_array)(attrib_position as u32); + (gl.gl_vertex_attrib_pointer)( + attrib_position as u32, + 2, + gl_const::GL_FLOAT, + gl_const::GL_FALSE, + stride, + std::ptr::null(), + ); + } + if attrib_tex_coord >= 0 { + (gl.gl_enable_vertex_attrib_array)(attrib_tex_coord as u32); + (gl.gl_vertex_attrib_pointer)( + attrib_tex_coord as u32, + 2, + gl_const::GL_FLOAT, + gl_const::GL_FALSE, + stride, + (2 * 4) as *const std::ffi::c_void, + ); + } + (gl.gl_draw_arrays)(gl_const::GL_TRIANGLE_STRIP, 0, 4); + (gl.gl_delete_buffers)(1, &vbo); + } +} + +/// 在设备中查找可用的中文字体（优先 Noto Sans/Serif CJK）。 +fn find_cjk_font() -> Option { + let candidates = [ + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc", + "/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc", + ]; + candidates + .iter() + .map(Path::new) + .find(|p| p.exists()) + .map(|p| p.to_path_buf()) +} diff --git a/src/plugins/live2d/text.rs b/src/plugins/live2d/text.rs new file mode 100644 index 0000000..ef256e5 --- /dev/null +++ b/src/plugins/live2d/text.rs @@ -0,0 +1,239 @@ +//! 基于 FreeType 的中文文字光栅化 + OpenGL 纹理渲染（设备端 Live2D 字幕 overlay）。 +//! +//! 设计：每个字符首次出现时用 FreeType 渲染为灰度位图（经 `render_glyph` 拿到位图），转成 +//! 白色字形（alpha=灰度）的 RGBA 纹理并缓存；绘制时按 pen 位置贴纹理即可。纹理创建需要 +//! `Gl` 函数指针（与渲染器共用同一 EGL 上下文）。 + +use super::gl::gl as gl_const; +use super::gl::Gl; +use freetype::face::LoadFlag; +use freetype::Face; +use freetype::Library; +use freetype::RenderMode; +use std::collections::HashMap; +use std::path::Path; + +use anyhow::Result; + +/// 单个字符的缓存字形 +#[derive(Clone)] +pub struct CachedGlyph { + /// 字形纹理（0 表示该字符无可见像素，仅用于步进） + pub tex: gl_const::GLuint, + pub w: i32, + pub h: i32, + /// 字形位图相对 pen 原点的偏移 + pub left: i32, + pub top: i32, + /// 26.6 定点像素步进（渲染端需 >> 6 转为像素） + pub advance: i32, +} + +pub struct TextRenderer { + face: Face<'static>, + font_px: u32, + glyphs: HashMap, + white_tex: Option, +} + +// FreeType 的 Face 包裹裸指针，仅在本插件单一渲染线程使用。 +unsafe impl Send for TextRenderer {} + +/// FT_LOAD_RENDER 标志位（0x4）：加载字符的同时渲染为位图 +const FT_LOAD_RENDER: i32 = 0x4; + +impl TextRenderer { + pub fn new(font_path: &Path, font_px: u32) -> Result { + let library = Library::init()?; + // 让 Library 成为 'static（程序级单例），从而 Face<'static> 可长期持有而不会悬垂 + let library: &'static Library = Box::leak(Box::new(library)); + let face = library.new_face(font_path, 0)?; + let tr = Self { + face, + font_px, + glyphs: HashMap::new(), + white_tex: None, + }; + tr.face.set_pixel_sizes(0, font_px)?; + println!("[Live2D] 字幕字体就绪: {:?} ({}px)", font_path, font_px); + Ok(tr) + } + + /// 当前字号（像素） + pub fn font_size(&self) -> u32 { + self.font_px + } + + /// 1x1 白色纹理（用于纯色背景框，alpha 恒为 1）。 + fn white_tex(&mut self, gl: &Gl) -> gl_const::GLuint { + if let Some(t) = self.white_tex { + return t; + } + let mut tex = 0u32; + unsafe { + (gl.gl_gen_textures)(1, &mut tex); + (gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_MIN_FILTER, + gl_const::GL_LINEAR as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_MAG_FILTER, + gl_const::GL_LINEAR as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_WRAP_S, + gl_const::GL_CLAMP_TO_EDGE as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_WRAP_T, + gl_const::GL_CLAMP_TO_EDGE as i32, + ); + let px: [u8; 4] = [255, 255, 255, 255]; + (gl.gl_tex_image_2d)( + gl_const::GL_TEXTURE_2D, + 0, + gl_const::GL_RGBA as i32, + 1, + 1, + 0, + gl_const::GL_RGBA, + gl_const::GL_UNSIGNED_BYTE, + px.as_ptr() as *const std::ffi::c_void, + ); + } + self.white_tex = Some(tex); + tex + } + + /// 取 1x1 白纹理（供外部画背景框）。 + pub fn white_texture(&mut self, gl: &Gl) -> gl_const::GLuint { + self.white_tex(gl) + } + + /// 估算字符像素宽度（用于换行布局，不创建纹理）。返回像素值。 + pub fn measure_advance(&mut self, ch: char) -> i32 { + if self + .face + .load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER)) + .is_ok() + { + (self.face.glyph().advance().x >> 6) as i32 + } else { + (self.font_px as f32 * 0.5) as i32 + } + } + + fn rasterize(&mut self, gl: &Gl, ch: char) -> Option { + let half_advance_26_6 = (self.font_px as i32) << 5; // 半字宽，26.6 定点 + if self + .face + .load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER)) + .is_err() + { + return Some(CachedGlyph { + tex: 0, + w: 0, + h: 0, + left: 0, + top: 0, + advance: half_advance_26_6, + }); + } + let glyph = self.face.glyph(); + if glyph.render_glyph(RenderMode::Normal).is_err() { + let adv = glyph.advance().x as i32; // 26.6 定点 + return Some(CachedGlyph { + tex: 0, + w: 0, + h: 0, + left: 0, + top: 0, + advance: adv, + }); + } + let bitmap = glyph.bitmap(); + let w = bitmap.width(); + let h = bitmap.rows(); + let left = glyph.bitmap_left(); + let top = glyph.bitmap_top(); + let advance = glyph.advance().x as i32; // 26.6 定点 + if w == 0 || h == 0 { + return Some(CachedGlyph { + tex: 0, + w: 0, + h: 0, + left, + top, + advance, + }); + } + // 灰度 -> RGBA：白色字形，alpha=灰度 + let buf = bitmap.buffer(); + let mut rgba: Vec = Vec::with_capacity((w * h * 4) as usize); + for &g in buf { + rgba.push(255); + rgba.push(255); + rgba.push(255); + rgba.push(g); + } + let mut tex = 0u32; + unsafe { + (gl.gl_gen_textures)(1, &mut tex); + (gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_MIN_FILTER, + gl_const::GL_LINEAR as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_MAG_FILTER, + gl_const::GL_LINEAR as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_WRAP_S, + gl_const::GL_CLAMP_TO_EDGE as i32, + ); + (gl.gl_tex_parameteri)( + gl_const::GL_TEXTURE_2D, + gl_const::GL_TEXTURE_WRAP_T, + gl_const::GL_CLAMP_TO_EDGE as i32, + ); + (gl.gl_tex_image_2d)( + gl_const::GL_TEXTURE_2D, + 0, + gl_const::GL_RGBA as i32, + w, + h, + 0, + gl_const::GL_RGBA, + gl_const::GL_UNSIGNED_BYTE, + rgba.as_ptr() as *const std::ffi::c_void, + ); + } + Some(CachedGlyph { + tex, + w, + h, + left, + top, + advance, + }) + } + + /// 获取（必要时光栅化并缓存）字符字形。 + pub fn glyph(&mut self, gl: &Gl, ch: char) -> Option { + if !self.glyphs.contains_key(&ch) { + if let Some(g) = self.rasterize(gl, ch) { + self.glyphs.insert(ch, g); + } + } + self.glyphs.get(&ch).cloned() + } +}