pulsareon/ShowenV2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add device-side subtitle overlay using FreeType for rendering Chinese text

Browse Source 
- Updated Cargo.toml to include freetype-rs for font rendering.
- Modified message.rs to introduce a new Message variant for subtitles.
- Enhanced HttpPlugin to send subtitle messages to the live2d plugin.
- Implemented text rendering in the live2d plugin using FreeType for Chinese characters.
- Added OpenGL shaders for rendering text overlays.
- Created a new text.rs module for handling text rasterization and texture rendering.
- Updated renderer.rs to integrate subtitle rendering logic.
This commit is contained in:
XiuchengWu
2026-07-09 09:15:23 +08:00
parent 9e31aee42d
commit 7cb8cee70d
10 changed files with 717 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
Cargo.lock generated
View File

@@ -41,6 +41,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1370e9fc2a6ae53aea8b7a5110edbd08836ed87c88736dfabccade1c2b44bff4"
[[package]]
name = "bitflags"
version = "2.11.0"
@@ -215,7 +221,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"block2",
"libc",
"objc2",
@@ -314,6 +320,28 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "freetype-rs"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1418e2a055fec8efe18c1a90a54b2cf5e649e583830dd4c71226c4e3bc920c6"
dependencies = [
"bitflags 0.8.2",
"freetype-sys",
"libc",
]
[[package]]
name = "freetype-sys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eccfb6d96cac99921f0c2142a91765f6c219868a2c45bdfe7d65a08775f18127"
dependencies = [
"libc",
"libz-sys",
"pkg-config",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
@@ -695,12 +723,24 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"libc",
"plain",
"redox_syscall",
]
[[package]]
name = "libz-sys"
version = "1.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -802,7 +842,7 @@ version = "0.31.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"cfg-if",
"cfg_aliases",
"libc",
@@ -924,7 +964,7 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"crc32fast",
"fdeflate",
"flate2",
@@ -1015,7 +1055,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16"
dependencies = [
"bitflags",
"bitflags 2.11.0",
]
[[package]]
@@ -1067,7 +1107,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"errno",
"libc",
"linux-raw-sys",
@@ -1226,6 +1266,7 @@ dependencies = [
"dbus",
"dbus-crossroads",
"flate2",
"freetype-rs",
"futures-util",
"image",
"libloading",

2
Cargo.toml
View File

@@ -40,3 +40,5 @@ semver = "1"
# Live2D 渲染
image = { version = "0.25", default-features = false, features = ["png"] }
# 设备端字幕（中文光栅化，调用系统 FreeType 高级封装）
freetype-rs = "0.13"

4
src/core/message.rs
View File

@@ -69,6 +69,10 @@ pub enum Message {
/// AI 模型状态变更广播（加载/切换/下载进度）
AiModelEvent(AiModelEvent),
// ── 设备端字幕 (M2.1) ──
/// 设备端 Live2D 字幕显示（AI 回复文字 Overlay）；HTTP 插件发出，live2d 插件消费
Subtitle { text: String },
// ── 扩展（未来插件用） ──
Custom {
kind: String,

24
src/plugins/http/mod.rs
View File

@@ -5,11 +5,12 @@
mod routes;
use crate::core::config::AppConfig;
use crate::core::message::{Envelope, Message};
use crate::core::message::{Destination, Envelope, Message};
use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo};
use anyhow::{Context, Result};
use serde::Serialize;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::sync::{Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use tokio::sync::broadcast;
@@ -54,10 +55,12 @@ pub(crate) struct HttpState {
ai_models: Mutex<Option<std::sync::Arc<std::sync::Mutex<crate::plugins::ai::model_manager::ModelManager>>>>,
/// Live2D 说话状态（设备端 Firefox 轮询，驱动嘴部动效）
live2d_talking: AtomicBool,
/// 向其他插件（如 live2d）转发消息的出口（init 时由 PluginContext.tx 注入）
outbox: mpsc::Sender<Envelope>,
}
impl HttpState {
fn new(config: Arc<AppConfig>) -> Self {
fn new(config: Arc<AppConfig>, tx: mpsc::Sender<Envelope>) -> Self {
let (ws_events, _) = broadcast::channel(32);
let player_status = crate::core::message::PlayerStatusData {
running: false,
@@ -84,6 +87,18 @@ impl HttpState {
ai_pipeline: Mutex::new(None),
ai_models: Mutex::new(None),
live2d_talking: AtomicBool::new(false),
outbox: tx,
}
}
/// 向指定插件发送消息（用于把 AI 回复文字推给 live2d 做设备端字幕）。
pub(crate) fn send_message(&self, to: Destination, msg: Message) {
if let Err(e) = self.outbox.send(Envelope {
from: "http".to_string(),
to,
message: msg,
}) {
eprintln!("[HttpPlugin] 发送消息失败: {e}");
}
}
@@ -312,7 +327,10 @@ impl Plugin for HttpPlugin {
}
fn init(&mut self, ctx: PluginContext) -> Result<()> {
let state = Arc::new(HttpState::new(Arc::clone(&ctx.config)));
let state = Arc::new(HttpState::new(
Arc::clone(&ctx.config),
ctx.tx.clone(),
));
// 注册 AI 句柄（如果 main.rs 已设置）
if let Some(pipeline) = self.ai_pipeline.take() {
state.register_ai_pipeline(pipeline);

20
src/plugins/http/routes.rs
View File

@@ -2188,6 +2188,16 @@ fn chat_text_route(
reset_state.set_live2d_talking(false);
});
// 设备端字幕：把回复文字推给 live2d 插件做屏幕 Overlay
if resp.error.is_none() && !resp.reply_text.is_empty() {
state.send_message(
Destination::Plugin("live2d".to_string()),
Message::Subtitle {
text: resp.reply_text.clone(),
},
);
}
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {
@@ -2274,6 +2284,16 @@ fn chat_audio_route(
// 清理上传的临时音频
let _ = std::fs::remove_file(&audio_path_clone);
// 设备端字幕：把回复文字推给 live2d 插件做屏幕 Overlay
if resp.error.is_none() && !resp.reply_text.is_empty() {
state.send_message(
Destination::Plugin("live2d".to_string()),
Message::Subtitle {
text: resp.reply_text.clone(),
},
);
}
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {

93
src/plugins/live2d/gl.rs
View File

@@ -205,6 +205,29 @@ void main() {
}
"#;
/// 文字 overlay 着色器（像素坐标 -> 裁剪空间，采样字形纹理，按 u_color 着色）
pub const TEXT_VERT_SHADER_SRC: &str = r#"#version 100
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform mat4 u_matrix;
varying vec2 v_uv;
void main() {
gl_Position = u_matrix * vec4(a_position, 0.0, 1.0);
v_uv = a_texCoord;
}
"#;
pub const TEXT_FRAG_SHADER_SRC: &str = r#"#version 100
precision mediump float;
varying vec2 v_uv;
uniform sampler2D s_texture0;
uniform vec4 u_color;
void main() {
float a = texture2D(s_texture0, v_uv).a;
gl_FragColor = vec4(u_color.rgb, a * u_color.a);
}
"#;
/// 遮罩（mask）着色器：仅把 mask drawable 的不透明区域写入 stencil，
/// 用于裁切被遮罩的 drawable（如夹在身体里的后臂、被头发遮挡的脸）。
pub const MASK_VERT_SHADER_SRC: &str = r#"#version 100
@@ -255,6 +278,12 @@ pub struct Gl {
config_size: egl::EGLint,
num_config: *mut egl::EGLint,
) -> egl::EGLBoolean,
pub egl_get_config_attrib: unsafe extern "C" fn(
dpy: egl::EGLDisplay,
config: egl::EGLConfig,
attribute: egl::EGLint,
value: *mut egl::EGLint,
) -> egl::EGLBoolean,
pub egl_create_window_surface: unsafe extern "C" fn(
dpy: egl::EGLDisplay,
config: egl::EGLConfig,
@@ -433,6 +462,7 @@ impl Gl {
egl_get_display: *lib_egl.get(b"eglGetDisplay\0")?,
egl_initialize: *lib_egl.get(b"eglInitialize\0")?,
egl_choose_config: *lib_egl.get(b"eglChooseConfig\0")?,
egl_get_config_attrib: *lib_egl.get(b"eglGetConfigAttrib\0")?,
egl_create_window_surface: *lib_egl.get(b"eglCreateWindowSurface\0")?,
egl_create_context: *lib_egl.get(b"eglCreateContext\0")?,
egl_make_current: *lib_egl.get(b"eglMakeCurrent\0")?,
@@ -530,6 +560,13 @@ pub struct RenderContext {
pub mask_uniform_texture: GLint,
pub mask_uniform_multiply_color: GLint,
pub mask_uniform_screen_color: GLint,
// 文字 overlay 程序
pub text_program: GLuint,
pub text_uniform_matrix: GLint,
pub text_uniform_color: GLint,
pub text_uniform_texture: GLint,
pub text_attrib_position: GLint,
pub text_attrib_tex_coord: GLint,
pub width: i32,
pub height: i32,
}
@@ -714,6 +751,18 @@ impl RenderContext {
num_config
);
// 真实查询 chosen config 的 depth/stencil 位数，确认 stencil 缓冲真正分配。
let mut stencil_size: egl::EGLint = -1;
let mut depth_size: egl::EGLint = -1;
unsafe {
(gl.egl_get_config_attrib)(display, config, egl::EGL_STENCIL_SIZE, &mut stencil_size);
(gl.egl_get_config_attrib)(display, config, egl::EGL_DEPTH_SIZE, &mut depth_size);
}
println!(
"[Live2D] EGL 实际分配: depth={} stencil={} (stencil<=0 表示遮罩不可用)",
depth_size, stencil_size
);
let surface = unsafe {
(gl.egl_create_window_surface)(display, config, x_window as egl::NativeWindowType, std::ptr::null())
};
@@ -846,6 +895,44 @@ impl RenderContext {
unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) }
};
// 文字 overlay 程序
let text_vert = compile_shader(&gl, gl::GL_VERTEX_SHADER, TEXT_VERT_SHADER_SRC)?;
let text_frag = compile_shader(&gl, gl::GL_FRAGMENT_SHADER, TEXT_FRAG_SHADER_SRC)?;
let text_program = unsafe {
let p = (gl.gl_create_program)();
(gl.gl_attach_shader)(p, text_vert);
(gl.gl_attach_shader)(p, text_frag);
(gl.gl_link_program)(p);
let mut status = 0;
(gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status);
if status == 0 {
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
return Err(anyhow!("text shader link failed: {}", String::from_utf8_lossy(&log)));
}
p
};
let text_uniform_matrix = {
let c = std::ffi::CString::new("u_matrix")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_uniform_color = {
let c = std::ffi::CString::new("u_color")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_uniform_texture = {
let c = std::ffi::CString::new("s_texture0")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_attrib_position = {
let c = std::ffi::CString::new("a_position")?;
unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) }
};
let text_attrib_tex_coord = {
let c = std::ffi::CString::new("a_texCoord")?;
unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) }
};
println!(
"[Live2D] GL 程序就绪: program={} pos={} tex={} mat={} tex_u={} base={} mult={} scr={}",
program, attrib_position, attrib_tex_coord, uniform_matrix,
@@ -882,6 +969,12 @@ impl RenderContext {
mask_uniform_texture,
mask_uniform_multiply_color,
mask_uniform_screen_color,
text_program,
text_uniform_matrix,
text_uniform_color,
text_uniform_texture,
text_attrib_position,
text_attrib_tex_coord,
width: win_w,
height: win_h,
})

1
src/plugins/live2d/mod.rs
View File

@@ -16,6 +16,7 @@ pub mod gl;
pub mod model;
pub mod pose;
pub mod renderer;
pub mod text;
mod plugin;
pub use plugin::Live2DPlugin;

27
src/plugins/live2d/plugin.rs
View File

@@ -9,10 +9,11 @@ use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo};
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use super::renderer::Live2DRenderer;
use super::renderer::{Live2DRenderer, Subtitle, SubtitleState};
pub struct Live2DPlugin {
ctx: Option<PluginContext>,
@@ -21,6 +22,8 @@ pub struct Live2DPlugin {
worker: Option<JoinHandle<()>>,
/// 渲染停止信号
stop_flag: Arc<AtomicBool>,
/// 字幕共享状态（http 插件写入，渲染线程读取）
subtitle: SubtitleState,
}
impl Live2DPlugin {
@@ -30,6 +33,7 @@ impl Live2DPlugin {
renderer: None,
worker: None,
stop_flag: Arc::new(AtomicBool::new(false)),
subtitle: Arc::new(Mutex::new(None)),
}
}
}
@@ -97,6 +101,16 @@ impl Plugin for Live2DPlugin {
Message::Shutdown => {
self.stop_live2d();
}
Message::Subtitle { text } => {
// 设备端字幕：写入共享状态，渲染线程每帧读取，超时自动消失
let ttl = Duration::from_millis(
(text.chars().count().max(1) as u64)
.saturating_mul(220)
.saturating_add(2500),
);
*self.subtitle.lock().unwrap() =
Some(Subtitle { text, expires_at: Instant::now() + ttl });
}
_ => {}
}
Ok(())
@@ -139,7 +153,14 @@ impl Live2DPlugin {
.clone();
let renderer = Arc::new(std::sync::Mutex::new(
Live2DRenderer::new(&core_so, &live2d_base, &model_rel, width, height)?,
Live2DRenderer::new(
&core_so,
&live2d_base,
&model_rel,
width,
height,
Arc::clone(&self.subtitle),
)?,
));
self.renderer = Some(Arc::clone(&renderer));

268
src/plugins/live2d/renderer.rs
View File

@@ -7,11 +7,26 @@ use super::gl::{self as gl_api, RenderContext};
use gl_api::gl as gl_const;
use super::model::{CubismModel, DrawableSnapshot};
use super::pose::PoseState;
use super::text::TextRenderer;
use anyhow::Result;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::{Duration, Instant};
/// 前几帧打印被遮罩 drawable 的 mask 引用关系，便于排查遮罩错误（如后臂整只伸开）。
static LIVE2D_MASK_DEBUG: AtomicU32 = AtomicU32::new(0);
/// 设备端字幕：AI 回复文字 + 自动消失时间。
pub struct Subtitle {
pub text: String,
pub expires_at: Instant,
}
/// 字幕共享状态（http 插件写入，live2d 渲染线程读取）。
pub type SubtitleState = Arc<Mutex<Option<Subtitle>>>;
pub struct Live2DRenderer {
pub core: Arc<CubismCore>,
pub model: CubismModel,
@@ -23,6 +38,10 @@ pub struct Live2DRenderer {
pub pose: Option<PoseState>,
pub model_matrix: [f32; 16],
pub running: bool,
/// 字幕共享状态（由 live2d 插件从 http 插件接收并传入）
pub subtitle: SubtitleState,
/// 中文字幕渲染器（无可用字体时为 None）
pub text: Option<TextRenderer>,
}
impl Live2DRenderer {
@@ -32,6 +51,7 @@ impl Live2DRenderer {
model3_json_rel: &str,
width: i32,
height: i32,
subtitle: SubtitleState,
) -> Result<Self> {
let core = Arc::new(CubismCore::load(core_so_path)?);
println!("[Live2D] Core version: {}", core.version_string());
@@ -68,6 +88,21 @@ impl Live2DRenderer {
pose.reset(&core, &model);
}
// 初始化中文字幕渲染器（找不到字体则降级为无字幕）
let text = match find_cjk_font() {
Some(p) => match TextRenderer::new(p.as_path(), 44) {
Ok(t) => Some(t),
Err(e) => {
eprintln!("[Live2D] 字幕渲染器初始化失败，字幕功能禁用: {e}");
None
}
},
None => {
eprintln!("[Live2D] 未找到中文字体，字幕功能禁用");
None
}
};
Ok(Self {
core,
model,
@@ -79,6 +114,8 @@ impl Live2DRenderer {
pose,
model_matrix,
running: false,
subtitle,
text,
})
}
@@ -98,6 +135,39 @@ impl Live2DRenderer {
let snapshots = self.model.drawable_snapshots(&self.core);
// 前 1 帧打印全部 drawable 的概要 + 顶点包围盒，用于排查遮罩（[Live2D][maskdbg]）
let dbg = LIVE2D_MASK_DEBUG.fetch_add(1, Ordering::SeqCst);
if dbg < 1 {
eprintln!("[Live2D][maskdbg] total={}", snapshots.len());
for s in &snapshots {
let (mut minx, mut miny, mut maxx, mut maxy) =
(f32::MAX, f32::MAX, f32::MIN, f32::MIN);
for v in &s.vertex_positions {
minx = minx.min(v[0]);
miny = miny.min(v[1]);
maxx = maxx.max(v[0]);
maxy = maxy.max(v[1]);
}
let inv = (s.constant_flags & super::core_ffi::constant_flags::IS_INVERTED_MASK)
!= 0;
eprintln!(
"[Live2D][maskdbg] i={} ro={} op={:.2} fl={:#04x} masked={} masks={:?} inv={} bbox=({:.0},{:.0})-({:.0},{:.0}) tex={}",
s.index,
s.render_order,
s.opacity,
s.constant_flags,
!s.mask_indices.is_empty(),
s.mask_indices,
inv,
minx,
miny,
maxx,
maxy,
s.texture_index
);
}
}
unsafe {
let gl = &self.gl_ctx.gl;
// 每帧校正 viewport 为窗口实际尺寸，避免窗口被 WM 调整大小后
@@ -135,6 +205,9 @@ impl Live2DRenderer {
}
}
// 设备端字幕 overlay（AI 回复文字）
self.draw_subtitle();
self.model.reset_dynamic_flags(&self.core);
unsafe {
@@ -259,7 +332,7 @@ impl Live2DRenderer {
(gl.gl_clear)(gl_const::GL_STENCIL_BUFFER_BIT);
(gl.gl_color_mask)(0, 0, 0, 0);
(gl.gl_stencil_func)(gl_const::GL_ALWAYS, 1, 0xFF);
(gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_REPLACE);
(gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_INCR);
(gl.gl_use_program)(self.gl_ctx.mask_program);
(gl.gl_uniform_matrix4fv)(
@@ -384,6 +457,117 @@ impl Live2DRenderer {
}
}
/// 绘制设备端字幕：屏幕底部半透明背景框 + 白色中文（AI 回复文字）。
fn draw_subtitle(&mut self) {
let (text, expired) = {
let guard = self.subtitle.lock().unwrap();
match guard.as_ref() {
Some(s) => (s.text.clone(), s.expires_at <= Instant::now()),
None => return,
}
};
if expired || text.trim().is_empty() {
return;
}
let text_renderer = match self.text.as_mut() {
Some(t) => t,
None => return,
};
let gl = &self.gl_ctx.gl;
// 复制 text program 的 uniform/attrib 位置到局部，避免与 &mut text_renderer 借位冲突
let prog = self.gl_ctx.text_program;
let u_mat = self.gl_ctx.text_uniform_matrix;
let u_col = self.gl_ctx.text_uniform_color;
let u_tex = self.gl_ctx.text_uniform_texture;
let a_pos = self.gl_ctx.text_attrib_position;
let a_tco = self.gl_ctx.text_attrib_tex_coord;
unsafe {
(gl.gl_enable)(gl_const::GL_BLEND);
(gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE_MINUS_SRC_ALPHA);
(gl.gl_use_program)(prog);
let w = self.gl_ctx.width as f32;
let h = self.gl_ctx.height as f32;
// 像素坐标（原点左上，y 向下）-> 裁剪空间
let ortho: [f32; 16] = [
2.0 / w, 0.0, 0.0, 0.0,
0.0, -2.0 / h, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
-1.0, 1.0, 0.0, 1.0,
];
(gl.gl_uniform_matrix4fv)(u_mat, 1, gl_const::GL_FALSE, ortho.as_ptr());
}
let font_px = text_renderer.font_size() as f32;
let line_height = (font_px * 1.35) as i32;
let margin = 40i32;
let max_width = self.gl_ctx.width - margin * 2;
let padding = 18i32;
// 按宽度换行（CJK 任意位置可断，遇 \n 强制断）
let mut lines: Vec<String> = Vec::new();
let mut cur = String::new();
let mut cur_w = 0i32;
for ch in text.chars() {
if ch == '\n' {
lines.push(std::mem::take(&mut cur));
cur_w = 0;
continue;
}
let aw = text_renderer.measure_advance(ch);
if cur_w + aw > max_width && !cur.is_empty() {
lines.push(std::mem::take(&mut cur));
cur_w = 0;
}
cur.push(ch);
cur_w += aw;
}
if !cur.is_empty() {
lines.push(cur);
}
if lines.is_empty() {
return;
}
let total_text_h = lines.len() as i32 * line_height;
let box_h = total_text_h + padding * 2;
let box_y = self.gl_ctx.height - margin - box_h;
let box_x = margin;
let box_w = self.gl_ctx.width - margin * 2;
// 背景框
let white = text_renderer.white_texture(gl);
draw_text_quad(
gl, white, box_x as f32, box_y as f32, box_w as f32, box_h as f32, [0.0, 0.0, 0.0, 0.55],
u_tex, u_col, a_pos, a_tco,
);
// 逐行绘制文字（居中）
for (li, line) in lines.iter().enumerate() {
let line_w: i32 = line.chars().map(|c| text_renderer.measure_advance(c)).sum();
let start_x = ((self.gl_ctx.width - line_w) / 2) as f32;
let baseline_y =
(box_y + padding + li as i32 * line_height + (line_height as f32 * 0.78) as i32) as f32;
let mut pen_x = start_x;
for ch in line.chars() {
if let Some(g) = text_renderer.glyph(gl, ch) {
if g.tex != 0 && g.w > 0 && g.h > 0 {
let gx = pen_x + g.left as f32;
let gy = baseline_y - g.top as f32;
draw_text_quad(
gl, g.tex, gx, gy, g.w as f32, g.h as f32, [1.0, 1.0, 1.0, 1.0],
u_tex, u_col, a_pos, a_tco,
);
}
pen_x += (g.advance >> 6) as f32;
} else {
pen_x += font_px * 0.5;
}
}
}
}
pub fn run_loop(&mut self) -> Result<()> {
self.running = true;
let frame_duration = Duration::from_millis(16);
@@ -442,3 +626,83 @@ fn compute_model_matrix(
0.0, 0.0, 0.0, 1.0,
]
}
/// 用 text program 画一个带纹理的四边形（TRIANGLE_STRIP）。
///
/// `color` 为 RGBA；着色器输出 = u_color.rgb 配合字形纹理的 alpha，因此背景框可传
/// 纯色 + alpha，字形可传白色 + alpha=1（字形纹理本身含 alpha 覆盖）。
fn draw_text_quad(
gl: &gl_api::Gl,
tex: gl_const::GLuint,
x: f32,
y: f32,
w: f32,
h: f32,
color: [f32; 4],
uniform_texture: gl_const::GLint,
uniform_color: gl_const::GLint,
attrib_position: gl_const::GLint,
attrib_tex_coord: gl_const::GLint,
) {
unsafe {
(gl.gl_active_texture)(gl_const::GL_TEXTURE0);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_uniform1i)(uniform_texture, 0);
(gl.gl_uniform4f)(uniform_color, color[0], color[1], color[2], color[3]);
let verts: [f32; 16] = [
x, y, 0.0, 0.0,
x + w, y, 1.0, 0.0,
x, y + h, 0.0, 1.0,
x + w, y + h, 1.0, 1.0,
];
let mut vbo = 0u32;
(gl.gl_gen_buffers)(1, &mut vbo);
(gl.gl_bind_buffer)(gl_const::GL_ARRAY_BUFFER, vbo);
(gl.gl_buffer_data)(
gl_const::GL_ARRAY_BUFFER,
(verts.len() * 4) as isize,
verts.as_ptr() as *const std::ffi::c_void,
gl_const::GL_STATIC_DRAW,
);
let stride = 4 * 4;
if attrib_position >= 0 {
(gl.gl_enable_vertex_attrib_array)(attrib_position as u32);
(gl.gl_vertex_attrib_pointer)(
attrib_position as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
std::ptr::null(),
);
}
if attrib_tex_coord >= 0 {
(gl.gl_enable_vertex_attrib_array)(attrib_tex_coord as u32);
(gl.gl_vertex_attrib_pointer)(
attrib_tex_coord as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
(2 * 4) as *const std::ffi::c_void,
);
}
(gl.gl_draw_arrays)(gl_const::GL_TRIANGLE_STRIP, 0, 4);
(gl.gl_delete_buffers)(1, &vbo);
}
}
/// 在设备中查找可用的中文字体（优先 Noto Sans/Serif CJK）。
fn find_cjk_font() -> Option<PathBuf> {
let candidates = [
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc",
"/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc",
"/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
];
candidates
.iter()
.map(Path::new)
.find(|p| p.exists())
.map(|p| p.to_path_buf())
}

239
src/plugins/live2d/text.rs Normal file
View File

@@ -0,0 +1,239 @@
//! 基于 FreeType 的中文文字光栅化 + OpenGL 纹理渲染（设备端 Live2D 字幕 overlay）。
//!
//! 设计：每个字符首次出现时用 FreeType 渲染为灰度位图（经 `render_glyph` 拿到位图），转成
//! 白色字形（alpha=灰度）的 RGBA 纹理并缓存；绘制时按 pen 位置贴纹理即可。纹理创建需要
//! `Gl` 函数指针（与渲染器共用同一 EGL 上下文）。
use super::gl::gl as gl_const;
use super::gl::Gl;
use freetype::face::LoadFlag;
use freetype::Face;
use freetype::Library;
use freetype::RenderMode;
use std::collections::HashMap;
use std::path::Path;
use anyhow::Result;
/// 单个字符的缓存字形
#[derive(Clone)]
pub struct CachedGlyph {
/// 字形纹理（0 表示该字符无可见像素，仅用于步进）
pub tex: gl_const::GLuint,
pub w: i32,
pub h: i32,
/// 字形位图相对 pen 原点的偏移
pub left: i32,
pub top: i32,
/// 26.6 定点像素步进（渲染端需 >> 6 转为像素）
pub advance: i32,
}
pub struct TextRenderer {
face: Face<'static>,
font_px: u32,
glyphs: HashMap<char, CachedGlyph>,
white_tex: Option<gl_const::GLuint>,
}
// FreeType 的 Face 包裹裸指针，仅在本插件单一渲染线程使用。
unsafe impl Send for TextRenderer {}
/// FT_LOAD_RENDER 标志位（0x4）：加载字符的同时渲染为位图
const FT_LOAD_RENDER: i32 = 0x4;
impl TextRenderer {
pub fn new(font_path: &Path, font_px: u32) -> Result<Self> {
let library = Library::init()?;
// 让 Library 成为 'static（程序级单例），从而 Face<'static> 可长期持有而不会悬垂
let library: &'static Library = Box::leak(Box::new(library));
let face = library.new_face(font_path, 0)?;
let tr = Self {
face,
font_px,
glyphs: HashMap::new(),
white_tex: None,
};
tr.face.set_pixel_sizes(0, font_px)?;
println!("[Live2D] 字幕字体就绪: {:?} ({}px)", font_path, font_px);
Ok(tr)
}
/// 当前字号（像素）
pub fn font_size(&self) -> u32 {
self.font_px
}
/// 1x1 白色纹理（用于纯色背景框，alpha 恒为 1）。
fn white_tex(&mut self, gl: &Gl) -> gl_const::GLuint {
if let Some(t) = self.white_tex {
return t;
}
let mut tex = 0u32;
unsafe {
(gl.gl_gen_textures)(1, &mut tex);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MIN_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MAG_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_S,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_T,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
let px: [u8; 4] = [255, 255, 255, 255];
(gl.gl_tex_image_2d)(
gl_const::GL_TEXTURE_2D,
0,
gl_const::GL_RGBA as i32,
1,
1,
0,
gl_const::GL_RGBA,
gl_const::GL_UNSIGNED_BYTE,
px.as_ptr() as *const std::ffi::c_void,
);
}
self.white_tex = Some(tex);
tex
}
/// 取 1x1 白纹理（供外部画背景框）。
pub fn white_texture(&mut self, gl: &Gl) -> gl_const::GLuint {
self.white_tex(gl)
}
/// 估算字符像素宽度（用于换行布局，不创建纹理）。返回像素值。
pub fn measure_advance(&mut self, ch: char) -> i32 {
if self
.face
.load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER))
.is_ok()
{
(self.face.glyph().advance().x >> 6) as i32
} else {
(self.font_px as f32 * 0.5) as i32
}
}
fn rasterize(&mut self, gl: &Gl, ch: char) -> Option<CachedGlyph> {
let half_advance_26_6 = (self.font_px as i32) << 5; // 半字宽，26.6 定点
if self
.face
.load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER))
.is_err()
{
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left: 0,
top: 0,
advance: half_advance_26_6,
});
}
let glyph = self.face.glyph();
if glyph.render_glyph(RenderMode::Normal).is_err() {
let adv = glyph.advance().x as i32; // 26.6 定点
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left: 0,
top: 0,
advance: adv,
});
}
let bitmap = glyph.bitmap();
let w = bitmap.width();
let h = bitmap.rows();
let left = glyph.bitmap_left();
let top = glyph.bitmap_top();
let advance = glyph.advance().x as i32; // 26.6 定点
if w == 0 || h == 0 {
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left,
top,
advance,
});
}
// 灰度 -> RGBA：白色字形，alpha=灰度
let buf = bitmap.buffer();
let mut rgba: Vec<u8> = Vec::with_capacity((w * h * 4) as usize);
for &g in buf {
rgba.push(255);
rgba.push(255);
rgba.push(255);
rgba.push(g);
}
let mut tex = 0u32;
unsafe {
(gl.gl_gen_textures)(1, &mut tex);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MIN_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MAG_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_S,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_T,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_image_2d)(
gl_const::GL_TEXTURE_2D,
0,
gl_const::GL_RGBA as i32,
w,
h,
0,
gl_const::GL_RGBA,
gl_const::GL_UNSIGNED_BYTE,
rgba.as_ptr() as *const std::ffi::c_void,
);
}
Some(CachedGlyph {
tex,
w,
h,
left,
top,
advance,
})
}
/// 获取（必要时光栅化并缓存）字符字形。
pub fn glyph(&mut self, gl: &Gl, ch: char) -> Option<CachedGlyph> {
if !self.glyphs.contains_key(&ch) {
if let Some(g) = self.rasterize(gl, ch) {
self.glyphs.insert(ch, g);
}
}
self.glyphs.get(&ch).cloned()
}
}