Compare commits

...

6 Commits

Author SHA1 Message Date
6f7e24db63 feat: M2.1 客户端骨架、BLE 修复、仓库整理与现状板
- BLE: 修复 BlueZ LocalName 与 Includes 冲突,串行注册与退避
- HTTP: 语音对话补 Live2D talking 标志与 session;Web 补 X-Session-Id
- Flutter: 角色/对话/模型页 + API,Gradle/依赖升级,麦克风权限
- 文档: 新增 docs/STATUS.md,校准 CLAUDE/PROGRESS/README
- 清理: 移除根目录截图与临时模型,运维脚本迁入 scripts/device
2026-07-09 12:41:34 +08:00
7cb8cee70d feat: add device-side subtitle overlay using FreeType for rendering Chinese text
- Updated Cargo.toml to include freetype-rs for font rendering.
- Modified message.rs to introduce a new Message variant for subtitles.
- Enhanced HttpPlugin to send subtitle messages to the live2d plugin.
- Implemented text rendering in the live2d plugin using FreeType for Chinese characters.
- Added OpenGL shaders for rendering text overlays.
- Created a new text.rs module for handling text rasterization and texture rendering.
- Updated renderer.rs to integrate subtitle rendering logic.
2026-07-09 09:15:23 +08:00
9e31aee42d feat(live2d): 添加 Pose 处理模块以管理互斥 Part 的显示状态,优化模型加载和参数更新 2026-07-09 00:29:56 +08:00
514fca6000 feat(renderer): 添加遮罩支持,优化绘制流程和背景渐变 2026-07-09 00:11:48 +08:00
547085c819 feat: 添加 Live2D 渲染插件及相关依赖,优化构建和启动脚本 2026-07-08 17:32:56 +08:00
a8d7c03139 fix(renderer): 修正注释中的中文翻译以提高可读性 2026-07-07 16:59:45 +08:00
58 changed files with 4482 additions and 461 deletions

View File

@@ -1 +0,0 @@
{"text":"hello","session_id":"e2e"}

26
.gitignore vendored
View File

@@ -4,14 +4,38 @@ target/
*.swo
*~
extern/
# 模型文件不进 git只保留下载链接在 model_manager.rs 清单里
model_download/
model_store/
# Live2D 模型文件不进 git体积大部署时单独下载到设备
configs/live2d/
# 测试脚本
# 测试/临时脚本与请求体
.test_*.sh
.chat_req.json
.commit_msg.txt
# 截图与调试 dump勿再提交到仓库根
*.xwd
*.png
*.jpg
!docs/**/*.png
# Flutter / Android 构建产物
clients/flutter/build/
clients/flutter/.dart_tool/
clients/flutter/.flutter-plugins-dependencies
clients/flutter/android/.gradle/
clients/flutter/android/local.properties
clients/flutter/**/GeneratedPluginRegistrant.java
*.apk
*.aab
*.ipa
# 本机 IDE / OS
.DS_Store
.idea/
.vscode/
*.iml

Binary file not shown.

View File

@@ -1,115 +0,0 @@
{
"Version": 3,
"FileReferences": {
"Moc": "Haru.moc3",
"Textures": [
"Haru.2048/texture_00.png",
"Haru.2048/texture_01.png"
],
"Physics": "Haru.physics3.json",
"Pose": "Haru.pose3.json",
"DisplayInfo": "Haru.cdi3.json",
"Expressions": [
{
"Name": "F01",
"File": "expressions/F01.exp3.json"
},
{
"Name": "F02",
"File": "expressions/F02.exp3.json"
},
{
"Name": "F03",
"File": "expressions/F03.exp3.json"
},
{
"Name": "F04",
"File": "expressions/F04.exp3.json"
},
{
"Name": "F05",
"File": "expressions/F05.exp3.json"
},
{
"Name": "F06",
"File": "expressions/F06.exp3.json"
},
{
"Name": "F07",
"File": "expressions/F07.exp3.json"
},
{
"Name": "F08",
"File": "expressions/F08.exp3.json"
}
],
"Motions": {
"Idle": [
{
"File": "motions/haru_g_idle.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5
},
{
"File": "motions/haru_g_m15.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5
}
],
"TapBody": [
{
"File": "motions/haru_g_m26.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5,
"Sound": "sounds/haru_talk_13.wav"
},
{
"File": "motions/haru_g_m06.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5,
"Sound": "sounds/haru_Info_14.wav"
},
{
"File": "motions/haru_g_m20.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5,
"Sound": "sounds/haru_normal_6.wav"
},
{
"File": "motions/haru_g_m09.motion3.json",
"FadeInTime": 0.5,
"FadeOutTime": 0.5,
"Sound": "sounds/haru_Info_04.wav"
}
]
},
"UserData": "Haru.userdata3.json"
},
"Groups": [
{
"Target": "Parameter",
"Name": "EyeBlink",
"Ids": [
"ParamEyeLOpen",
"ParamEyeROpen"
]
},
{
"Target": "Parameter",
"Name": "LipSync",
"Ids": [
"ParamMouthOpenY"
]
}
],
"HitAreas": [
{
"Id": "HitArea",
"Name": "Head"
},
{
"Id": "HitArea2",
"Name": "Body"
}
]
}

View File

@@ -90,14 +90,17 @@ export PATH="/home/showen/.rustup/toolchains/stable-aarch64-unknown-linux-gnu/bi
## 当前状态
- **质量**: **147/147 已在 Linux ARM64 目标机验证**2026-07-03 实跑lib 105 + example 7 + m1_2_dynamic 15 + m1_2_http 12 + m1_2_service_manager 8Flutter 15/15零 warning。**注**2026-07-04 新增 3 提交README 修正 + P1 Shutdown fix + P2 清理)仅过类型检查,待目标机复验
- **里程碑**: M1.1 完成,**M1.2 收尾完成**147/147 两轮验收全绿、风险3 关闭、老板已提交 2a6e14b、设备已同步并复验零 warning仅差老板打 tag建议在 2a6e14b 上打 `m1.2`
- **DevicePlugin 能力**: Display + SleepInhibit + Backlight + Cursor (Linux ARM64)
- **ScreenPlugin**: 已重构为 thin wrapper
- **Flutter App**: 完成度 ~98%, APK v0.3 (52.6MB) 已编译, cupertino_icons 已修复
- **API 文档**: 已校准(以 routes.rs 为唯一权威重写)
- **示例插件**: 已完善为开发者参考模板 (manifest.json + 请求/响应示范 + 7 个测试)
- **M1.2 进展**: 目标机两轮验收完成2026-07-03147/147 零 warning证据见 TEAM_CHAT.mdHTTP 路由测试实为 12 个(旧记录 9 有误、lib 单测实为 105旧记录 107 有误);剩余 commit+tag、风险3 补派核查
> **功能成熟度权威清单见 `docs/STATUS.md`2026-07-09 整理)**。此处只保留 CEO 一页摘要。
- **质量 (M1.2)**: **147/147 已在 Linux ARM64 目标机验证**lib 105 + example 7 + m1_2_dynamic 15 + m1_2_http 12 + m1_2_service_manager 8零 warning2026-07-04 三提交亦已复验全绿
- **里程碑**: M1.1 ✅ · M1.2 ✅(仅差老板打 tag `m1.2`)· **M2.1 🟡 半成品**
- **DevicePlugin**: Display + SleepInhibit + Backlight + Cursor (Linux ARM64)
- **插件**: video / http / ble / wifi / device / screen / **live2d** / **ai**
- **Flutter App**: M1 控制能力可用;**M2.1 角色/对话/模型页已写代码、debug APK 已出(~146MB真机验收未闭环** → 见 STATUS.md §3.3
- **Web 控制端**: 已切到 `chat.html`/`chat.js`(角色/对话/模型入口有);非旧 WEB_UI 常量
- **BLE**: 广播注册已修(不可 LocalName + Includes 双设);设备 ActiveInstances=1 曾验证
- **AI**: 设备文字对话 API 实测可通;延迟/模型档位/失败体验仍需产品化
- **本机开发环境 (Mac)**: Flutter 3.44.5 + JDK17 + Android SDK36`~/.showen_dev_env`
### 已修复(本轮)
@@ -148,26 +151,30 @@ export PATH="/home/showen/.rustup/toolchains/stable-aarch64-unknown-linux-gnu/bi
7. **Spike 结论2026-07-03 张明远实测,全表见 TEAM_CHAT.md**:可行。设备实为 Radxa Cubie A7AA7332×A78@2.0G + 6×A55@1.8G)。推荐默认 **Qwen2.5-0.5B Q4**16.4 t/s50字回复 ~3.4sRSS 985MGemma3-1B 可选档8.2 t/s**Gemma2-2B 本机不推荐**RSS 3.25G 触发 swap。硬约束**推理必须 t=2 锁大核**全核反而崩且挤死视频、严禁默认上下文长度、部署脚本必须带镜像回退。ASR whisper tiny ~2.2s(需 VAD 防静音幻觉、TTS piper RTF 0.3。端到端:串行热链 ~10s**流式管线首音频 ~5s**(达标"可接受"≤3s 需 A733 的 NPUPhase 2 路径)。现场保留 ~/ai_spike/ 3.5G 可复跑。
✅ PRD 完成docs/M2.1_PRD.md,三轮修订含老板三条补充)+ Spike 完成。进行中:架构设计(王思远)。素材依赖:数字人/歌姬内容包需视频素材,先用现有猫狗包验证切换机制。
✅ PRDdocs/M2.1_PRD.md+ Spike + 后端 AI/模型/角色 API + Web 控制台重构 + App M2.1 页面骨架。
**诚实结论**:能演示链路 ≠ 产品完成。详见 `docs/STATUS.md`
### 待处理
### 待处理(精简 · 2026-07-09
1. **[M1.2 收尾]** ✅ commit老板 2a6e14b→ ✅ 设备同步至 2a6e14b + cargo check 零 warning周雅婷 2026-07-03设备仅剩本地调参文件 configs/dog_state_machine.json 一处差异);**仅剩:老板打 tag建议在 2a6e14b 上打 `m1.2`**
2. ~~**[M1.2 收尾]** tests/m1_2_http.rs 过时注释更新~~ ✅ 赵雨薇完成2026-07-03纯注释改动核实点位后订正为"六种 plugin_* 闭环,唯 check_updates 结果回传属 Phase 2"注意此改动在第二轮验收之后commit 前设备端补一次 cargo check 即可
3. **[M1.2 收尾]** sanitize_filename 集成测试测的是逻辑复制而非产品函数,转正需产品侧小改动(改 pub 或 routes.rs 内加单测)— 赵雨薇上报
4. DevicePlugin 阶段三framebuffer迁移/触摸/音频/多平台)— Phase 2 规划
5. **[设备遗留-代码]** 单实例保护(锁文件/socket 互斥)+ unclutter 僵尸子进程 reap — 派内核/视频工程师
- ~~unclutter reap~~ ✅ 核查确认2026-07-04`linux_arm64.rs:238-254` shutdown() 已做 kill+wait+pkill 兜底,仅 SIGKILL 极端场景残留,不值得加 SIGCHLD handler
- **单实例保护**未做。需目标机实测锁文件行为Windows 开发环境无法验证留给设备端工程师。背景CLAUDE.md:136 记录的 3 实例 P0 已由赵雨薇移除多余自启入口修复,单实例锁是防御性双保险
6. ~~**[设备遗留-系统]** 时钟慢约一年 + 磁盘 9x% 满~~ ✅ 全部解决2026-07-03时钟 — 王浩然修复根因timesyncd 出厂未启用 + RTC hym8563 掉电失效标志;已配 aliyun NTP 持久化 + hwclock 恢复apt 复活重启验证开机即正确遗留RTC 备份电池未验证,冷启动需联网自愈);磁盘 — 赵雨薇两轮清理 96%→35%(删 Flutter/Android/编辑器/kilo 等 ~16G
7. **[环境]** 设备 SSH 已打通:`ssh -p 2222 showen@192.168.31.105` 免密2026-07-02 配置),目标机验收可远程执行
8. ~~**[代码质量-P2]** Shutdown 消息无生产触发者main.rs:117-120 Ctrl+C 只置 AtomicBool主循环阻塞在 rx.recv(),需等下一条任意消息才真正退出(王浩然发现)~~ ✅ 修复2026-07-04提交 ab1dd91三处改动 — run() recv()→recv_timeout(200ms)handle_manager_message Shutdown 分支置 self.running=falsemain.rs Ctrl+C handler 注入 Message::Shutdown。效果Ctrl+C 后最多 200ms 退出。**待目标机复验**
9. ~~**[Phase 2 清理]** SDK 文档教开发者监听永不触发的 WifiProvisioned (plugin-sdk/src/lib.rs:522)CursorVisibility 有消费无生产(实际链路已走 DeviceCommandcheck_updates 结果仅 println 不回传DeviceResponse 广播后核心无人读、Error 静默~~ ✅ 部分完成2026-07-04提交 38dcd21
- ✅ SDK 文档broadcast 示例换成 StateChanged新增 Phase 2 占位消息说明WifiProvisioned/DeviceEvent/CursorVisibility
- ✅ check_updates 结果回传:通过 Custom { kind: "plugin_updates" } 广播
- ✅ DeviceResponse::Error 留痕device/mod.rs handle_message 补 eprintln
- ⏳ CursorVisibility 变体:删除会破坏示例插件和测试,倾向 Phase 2 统一清理(保留现状)
10. ~~**[待复验]** 2026-07-04 三提交4d830c0 README / ab1dd91 P1 Shutdown fix / 38dcd21 P2 清理)仅过 rust-analyzer 类型检查,需在 Linux ARM64 目标机跑 `cargo check --workspace --all-targets && cargo test --workspace` 确认零 warning + 全测试通过~~ ✅ 目标机复验通过2026-07-04`cargo check` 零 warning`cargo test --workspace` 147/147 全过lib 105 + example 7 + m1_2_dynamic 15 + m1_2_http 12 + m1_2_service_manager 8P1 Shutdown 修复实测SIGINT 后 234ms 优雅退出exit_code=0修复前需等下一条消息可能永不退出
**P0 / M2.1 闭环**
1. **工作区提交**:本机有未提交的 BLE 修复 + HTTP 对话修补 + Flutter M2.1 页(老板提交,不代办)
2. **设备代码对齐 + 回归**:同步后 `cargo check/test --workspace`
3. **真机四旅程**:配网 → 角色切换 → 文字/语音对话 → 模型管理App + Web
4. **对话失败态产品化**AI 未就绪/超时/缺模型时客户端明确提示
**P1**
5. App 对话历史本地持久化、release APK
6. 模型下载进度与错误重试 UX
7. 数字人/歌姬素材PRD 外部依赖)
**P2 / 收尾**
8. **[M1.2]** 老板打 tag `m1.2`
9. sanitize_filename 测试转正单实例保护CursorVisibility 清理
10. DevicePlugin 阶段三framebuffer/触摸/音频/多平台)— 更后
**环境**
- 设备:`ssh -p 2222 showen@192.168.31.105` · Web `http://192.168.31.105:5000/`
- 本机 Flutter`source ~/.showen_dev_env`
---
@@ -228,12 +235,14 @@ export PATH="/home/showen/.rustup/toolchains/stable-aarch64-unknown-linux-gnu/bi
| 需要做什么 | 读什么文件 |
|-----------|-----------|
| **功能成熟度 / 不完善清单** | **`docs/STATUS.md`** |
| CEO 深层经验和管理方法论 | `souls/chen-yifei.md` |
| 查看团队最新动态/沟通 | `.showen/TEAM_CHAT.md` |
| 查看/修改公司详细规范 | `.showen/COMPANY_RULES.md` |
| 评估团队绩效/制度详情 | `docs/TEAM.md` |
| 查看工作流程/审核标准 | `docs/WORKFLOW.md` |
| 查看提交历史 | `PROGRESS.md` |
| 里程碑摘要 | `PROGRESS.md` |
| M2.1 产品需求 | `docs/M2.1_PRD.md` |
| 派发任务给 PM | `.showen/inbox/pm.md` |
| 查看某成员详情 | `souls/<name>.md` |
| 代码审核参考 | `docs/CODE_REVIEW.md` |

115
Cargo.lock generated
View File

@@ -41,6 +41,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1370e9fc2a6ae53aea8b7a5110edbd08836ed87c88736dfabccade1c2b44bff4"
[[package]]
name = "bitflags"
version = "2.11.0"
@@ -65,12 +71,24 @@ dependencies = [
"objc2",
]
[[package]]
name = "bytemuck"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "byteorder-lite"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
version = "1.11.1"
@@ -203,7 +221,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"block2",
"libc",
"objc2",
@@ -251,6 +269,15 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]]
name = "filetime"
version = "0.2.27"
@@ -293,6 +320,28 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "freetype-rs"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1418e2a055fec8efe18c1a90a54b2cf5e649e583830dd4c71226c4e3bc920c6"
dependencies = [
"bitflags 0.8.2",
"freetype-sys",
"libc",
]
[[package]]
name = "freetype-sys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eccfb6d96cac99921f0c2142a91765f6c219868a2c45bdfe7d65a08775f18127"
dependencies = [
"libc",
"libz-sys",
"pkg-config",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
@@ -604,6 +653,19 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "image"
version = "0.25.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
dependencies = [
"bytemuck",
"byteorder-lite",
"moxcms",
"num-traits",
"png",
]
[[package]]
name = "indexmap"
version = "2.13.0"
@@ -661,12 +723,24 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"libc",
"plain",
"redox_syscall",
]
[[package]]
name = "libz-sys"
version = "1.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -734,6 +808,16 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "moxcms"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
dependencies = [
"num-traits",
"pxfm",
]
[[package]]
name = "multer"
version = "2.1.0"
@@ -758,7 +842,7 @@ version = "0.31.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"cfg-if",
"cfg_aliases",
"libc",
@@ -874,6 +958,19 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "png"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
dependencies = [
"bitflags 2.11.0",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]]
name = "potential_utf"
version = "0.1.4"
@@ -901,6 +998,12 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "pxfm"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
[[package]]
name = "quote"
version = "1.0.45"
@@ -952,7 +1055,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16"
dependencies = [
"bitflags",
"bitflags 2.11.0",
]
[[package]]
@@ -1004,7 +1107,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"errno",
"libc",
"linux-raw-sys",
@@ -1163,7 +1266,9 @@ dependencies = [
"dbus",
"dbus-crossroads",
"flate2",
"freetype-rs",
"futures-util",
"image",
"libloading",
"opencv",
"rand",

View File

@@ -40,3 +40,5 @@ semver = "1"
# Live2D 渲染
image = { version = "0.25", default-features = false, features = ["png"] }
# 设备端字幕(中文光栅化,调用系统 FreeType 高级封装)
freetype-rs = "0.13"

View File

@@ -1,53 +1,44 @@
# ShowenV2 — 项目进度
> 当前状态和待办事项的权威来源是 `CLAUDE.md`。本文件保留里程碑摘要和最近变更。
> 完整提交历史见 `.showen/PROGRESS_ARCHIVE.md`。
> **功能成熟度看 `docs/STATUS.md`。**
> 派发/铁律/团队看 `CLAUDE.md`。
> 完整提交历史见 `.showen/PROGRESS_ARCHIVE.md`(若有)。
## 当前里程碑
**M1.2 — 测试验收已通过,收尾中** 🔄
- 集成测试全部编写完毕ServiceManager 8 + HTTP路由 12 + 动态插件 15 = 35 个)
- 2026-07-03 目标机Linux ARM64, 192.168.31.105)两轮验收:**147/147 全过、零 warning**(第一轮暴露 1 个测试前提错误 + 3 个 warning返工后第二轮全绿证据见 TEAM_CHAT.md
- 剩余commit 返工测试文件 + 打 tag待批准+ 风险3 生产者核查(王浩然)
| 里程碑 | 状态 |
|--------|------|
| M1.1 | ✅ 完成 |
| M1.2 | ✅ 验收通过147/147仅差 tag `m1.2` |
| **M2.1 语音数字生命 V1** | 🟡 **半成品** — 后端/Web/App 有骨架,真机验收与体验未闭环 |
---
## M2.1 一句话
**M1.1 — 完成**
- 30 个提交Phase 1 骨架 + 功能迁移 + 动态插件 + DevicePlugin 阶段一/二
- 115/115 测试通过107 核心 + 8 集成),零 warningM1.1结束时77/77M1.2期间增至115
- DevicePlugin: Display + SleepInhibit + Backlight + Cursor (Linux ARM64)
- ScreenPlugin 重构为 thin wrapper
本地 ASR→LLM→TTS声音出在 **App/Web**,设备负责算力和画面;角色可切换,模型可管理。
**现状**API 与 UI 入口大致齐了,**不能宣称功能完善**(见 STATUS.md 缺口表)。
## 最近变更 (提交 26-30)
## 最近本机工作(未全部入库)
| # | 提交 | 内容 | 负责人 |
|---|------|------|--------|
| 26 | `48d1eeb` | plugin-sdk 同步 Device 类型 | 李思琪 |
| 27 | `f060519` | DeviceCommand 添加 SetCursorVisible | 张明远 |
| 28 | `5310a92` | LinuxArm64Backend 添加光标控制 | 赵雨薇 |
| 29 | `bf41c45` | ScreenPlugin 重构为 thin wrapper | 赵雨薇 |
| 30 | `be08c63` | 新增 4 个光标控制集成测试 | 李思琪 |
- BLE修复 BlueZ `LocalName` + `Includes["local-name"]` 冲突;串行注册
- HTTP语音对话补 Live2D 说话态与 sessionWeb 补 `X-Session-Id`
- Flutter角色 / 对话 / 模型三页 + debug APK~146MB
- 仓库整理:根目录截图/临时模型清除;运维脚本迁 `scripts/device/`;新增 `docs/STATUS.md`
## 架构概览
```
core/ (插件微内核)
ServiceManager → 生命周期/消息路由/错误策略
Plugin trait → 统一插件接口
Message enum → 类型安全消息 (Serialize/Deserialize)
动态插件层 → FFI Loader / Runtime / Self-Test
plugins/ (一切皆插件)
video/ screen/ http/ ble/ wifi/ device/
(未来: render/ avatar/ vr/ ar/ voice/ ai/ singer/)
core/ 插件微内核 · 消息 · 配置 · 动态插件
plugins/ video · http · ble · wifi · device · screen · live2d · ai
configs/ 角色内容包 JSON
clients/ flutter 安卓控制端
scripts/device 设备运维脚本
docs/STATUS.md 功能成熟度板
```
## 实机验证结果
## 实机锚点
| 验证项 | 结果 |
|--------|------|
| `--validate` | 21 个视频路径全部有效 |
| 插件初始化 | 5/5 正常 start |
| HTTP API | `/api/status``/api/playlist` 正常 |
| framebuffer | fb0 480x800 检测成功 |
| Release 编译 | 9.4MB ARM aarch64 |
| 项 | |
|----|-----|
| 设备 | Radxa Cubie A7A · `192.168.31.105:2222` · showen |
| Web | `http://192.168.31.105:5000/` |
| 本机 APK | `clients/flutter/build/app/outputs/flutter-apk/app-debug.apk` |

View File

@@ -6,33 +6,20 @@
```
ShowenV2/
├── CLAUDE.md / docs/STATUS.md # 管理手册 · 功能成熟度板
├── src/
│ ├── core/ # 插件微内核
│ ├── message.rs # 类型安全消息协议 (Serialize/Deserialize)
├── plugin.rs # Plugin trait 定义
│ │ ├── service_manager.rs # 生命周期管理 + 消息路由 + 错误策略
│ │ ├── config.rs # 配置解析/验证
│ │ ├── dispatch.rs # 文本命令解析
├── plugin_abi.rs # C FFI 类型 (动态插件边界)
├── dynamic_plugin.rs # DynamicPlugin (libloading)
│ │ ├── plugin_loader.rs # plugin_store/ 扫描/加载
│ │ ├── version_manager.rs # 版本切换/回退/GC
│ │ └── plugin_repo.rs # 远程仓库 HTTP 客户端
│ └── plugins/ # 功能插件
│ ├── video/ # 视频播放引擎 (OpenCV)
│ ├── http/ # Web UI + REST API (warp)
│ ├── ble/ # BLE 配网 (D-Bus BlueZ)
│ ├── wifi/ # WiFi 管理 (nmcli)
│ ├── device/ # 设备能力 (Display/SleepInhibit/Backlight/Cursor)
│ └── screen/ # 屏幕管理 (thin wrapper转发到 device)
├── plugin-sdk/ # 动态插件开发 SDK
├── plugins/
│ └── example-plugin/ # 示例动态插件 (cdylib)
├── configs/ # 配置文件 (状态机 JSON)
├── clients/ # 外部控制客户端应用
└── souls/ # 团队成员档案
│ ├── core/ # 插件微内核(消息/配置/动态插件/版本)
└── plugins/ # video · http · ble · wifi · device · screen · live2d · ai
├── configs/ # 角色内容包 JSON
├── clients/flutter/ # 安卓控制 AppM1 遥控 + M2.1 对话骨架)
├── scripts/device/ # 设备运维脚本
├── plugin-sdk/ · plugins/ # 动态插件 SDK 与示例
├── tests/ # M1.2 集成测试
└── docs/ # PRD / 测试计划 / STATUS
```
**功能做到哪了**:请先读 [`docs/STATUS.md`](docs/STATUS.md),勿仅看提交日期。
## 技术栈
- **语言**: Rust (edition 2021)
@@ -47,7 +34,7 @@ ShowenV2/
ShowenV2 支持两种插件模式:
1. **静态插件** — 编译时链接6 个内置插件 (video, http, ble, wifi, device, screen)
1. **静态插件** — 编译时链接video, http, ble, wifi, device, screen, live2d, ai 等)
2. **动态插件** — 运行时加载 `.so` 文件,通过 `extern "C"` FFI + JSON 序列化通信
动态插件特性:

View File

@@ -7,6 +7,7 @@ plugins {
android {
namespace 'com.showen.flutter'
compileSdk flutter.compileSdkVersion
ndkVersion = "28.2.13676358"
defaultConfig {
applicationId 'com.showen.flutter'

View File

@@ -8,6 +8,8 @@
<uses-permission android:name="android.permission.BLUETOOTH_SCAN" />
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT" />
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
<application
android:label="ShowenV2"

View File

@@ -15,11 +15,31 @@ import io.flutter.embedding.engine.FlutterEngine;
public final class GeneratedPluginRegistrant {
private static final String TAG = "GeneratedPluginRegistrant";
public static void registerWith(@NonNull FlutterEngine flutterEngine) {
try {
flutterEngine.getPlugins().add(new xyz.luan.audioplayers.AudioplayersPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin audioplayers_android, xyz.luan.audioplayers.AudioplayersPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.lib.flutter_blue_plus.FlutterBluePlusPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin flutter_blue_plus_android, com.lib.flutter_blue_plus.FlutterBluePlusPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.github.dart_lang.jni.JniPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin jni, com.github.dart_lang.jni.JniPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.github.dart_lang.jni_flutter.JniFlutterPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin jni_flutter, com.github.dart_lang.jni_flutter.JniFlutterPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.llfbandit.record.RecordPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin record_android, com.llfbandit.record.RecordPlugin", e);
}
try {
flutterEngine.getPlugins().add(new io.flutter.plugins.sharedpreferences.SharedPreferencesPlugin());
} catch (Exception e) {

View File

@@ -1,3 +1,7 @@
org.gradle.jvmargs=-Xmx1536M -XX:MaxMetaspaceSize=512M -XX:+HeapDumpOnOutOfMemoryError
android.useAndroidX=true
android.enableJetifier=true
# This builtInKotlin flag was added automatically by Flutter migrator
android.builtInKotlin=false
# This newDsl flag was added automatically by Flutter migrator
android.newDsl=false

View File

@@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip

View File

@@ -21,8 +21,8 @@ pluginManagement {
plugins {
id 'dev.flutter.flutter-plugin-loader' version '1.0.0'
id 'com.android.application' version '8.3.2' apply false
id 'org.jetbrains.kotlin.android' version '1.9.22' apply false
id 'com.android.application' version '8.7.3' apply false
id 'org.jetbrains.kotlin.android' version '2.1.0' apply false
}
dependencyResolutionManagement {

View File

@@ -2,15 +2,21 @@ import 'package:flutter/material.dart';
import 'package:go_router/go_router.dart';
import 'package:provider/provider.dart';
import 'providers/character_provider.dart';
import 'providers/chat_provider.dart';
import 'providers/device_provider.dart';
import 'providers/debug_provider.dart';
import 'providers/ble_provider.dart';
import 'providers/model_provider.dart';
import 'providers/player_provider.dart';
import 'providers/wifi_provider.dart';
import 'screens/app_shell.dart';
import 'screens/ble_provision_screen.dart';
import 'screens/characters_screen.dart';
import 'screens/chat_screen.dart';
import 'screens/debug_screen.dart';
import 'screens/home_screen.dart';
import 'screens/models_screen.dart';
import 'screens/network_screen.dart';
import 'screens/playback_screen.dart';
import 'screens/settings_screen.dart';
@@ -36,6 +42,7 @@ Future<void> main() async {
runApp(
MultiProvider(
providers: [
Provider<HttpApiService>.value(value: httpApiService),
Provider<WebSocketService>.value(value: webSocketService),
ChangeNotifierProvider<DebugProvider>(
create: (_) => DebugProvider(webSocketService: webSocketService),
@@ -70,6 +77,24 @@ Future<void> main() async {
debugProvider: context.read<DebugProvider>(),
)..bootstrap(),
),
ChangeNotifierProvider<CharacterProvider>(
create: (context) => CharacterProvider(
httpApiService: httpApiService,
debugProvider: context.read<DebugProvider>(),
),
),
ChangeNotifierProvider<ChatProvider>(
create: (context) => ChatProvider(
httpApiService: httpApiService,
debugProvider: context.read<DebugProvider>(),
),
),
ChangeNotifierProvider<ModelProvider>(
create: (context) => ModelProvider(
httpApiService: httpApiService,
debugProvider: context.read<DebugProvider>(),
),
),
],
child: const ShowenApp(),
),
@@ -95,18 +120,27 @@ final GoRouter _router = GoRouter(
StatefulShellBranch(
routes: [
GoRoute(
path: '/playback',
name: 'playback',
builder: (context, state) => const PlaybackScreen(),
path: '/characters',
name: 'characters',
builder: (context, state) => const CharactersScreen(),
),
],
),
StatefulShellBranch(
routes: [
GoRoute(
path: '/trigger',
name: 'trigger',
builder: (context, state) => const TriggerScreen(),
path: '/chat',
name: 'chat',
builder: (context, state) => const ChatScreen(),
),
],
),
StatefulShellBranch(
routes: [
GoRoute(
path: '/playback',
name: 'playback',
builder: (context, state) => const PlaybackScreen(),
),
],
),
@@ -132,15 +166,23 @@ final GoRouter _router = GoRouter(
path: '/settings',
name: 'settings',
builder: (context, state) => const SettingsScreen(),
),
],
),
StatefulShellBranch(
routes: [
GoRoute(
path: '/debug',
name: 'debug',
builder: (context, state) => const DebugScreen(),
routes: [
GoRoute(
path: 'models',
name: 'models',
builder: (context, state) => const ModelsScreen(),
),
GoRoute(
path: 'trigger',
name: 'trigger',
builder: (context, state) => const TriggerScreen(),
),
GoRoute(
path: 'debug',
name: 'debug',
builder: (context, state) => const DebugScreen(),
),
],
),
],
),

View File

@@ -0,0 +1,123 @@
class AiModelInfo {
const AiModelInfo({
required this.id,
required this.kind,
required this.name,
required this.version,
required this.size,
required this.memoryRequired,
required this.recommendedTier,
required this.downloaded,
required this.downloadProgress,
});
final String id;
final String kind;
final String name;
final String version;
final int size;
final int memoryRequired;
final String recommendedTier;
final bool downloaded;
final int downloadProgress;
String get kindLabel {
switch (kind.toLowerCase()) {
case 'llm':
return 'LLM';
case 'asr':
return 'ASR';
case 'tts':
return 'TTS';
default:
return kind.toUpperCase();
}
}
String get sizeLabel => _formatBytes(size);
String get memoryLabel => _formatBytes(memoryRequired);
factory AiModelInfo.fromJson(Map<String, dynamic> json) {
return AiModelInfo(
id: json['id']?.toString() ?? '',
kind: json['kind']?.toString() ?? '',
name: json['name']?.toString() ?? json['id']?.toString() ?? '',
version: json['version']?.toString() ?? '',
size: _asInt(json['size']),
memoryRequired: _asInt(json['memory_required']),
recommendedTier: json['recommended_tier']?.toString() ?? '',
downloaded: json['downloaded'] == true,
downloadProgress: _asInt(json['download_progress']),
);
}
static int _asInt(dynamic value) {
if (value is int) return value;
if (value is num) return value.toInt();
return int.tryParse(value?.toString() ?? '') ?? 0;
}
static String _formatBytes(int bytes) {
if (bytes <= 0) return '0 B';
if (bytes < 1024) return '$bytes B';
if (bytes < 1024 * 1024) {
return '${(bytes / 1024).toStringAsFixed(1)} KB';
}
if (bytes < 1024 * 1024 * 1024) {
return '${(bytes / (1024 * 1024)).toStringAsFixed(1)} MB';
}
return '${(bytes / (1024 * 1024 * 1024)).toStringAsFixed(2)} GB';
}
}
class ModelsSnapshot {
const ModelsSnapshot({
required this.models,
required this.active,
required this.usedSpace,
required this.quota,
});
final List<AiModelInfo> models;
final Map<String, String> active;
final int usedSpace;
final int quota;
String get usedLabel => AiModelInfo._formatBytes(usedSpace);
String get quotaLabel => AiModelInfo._formatBytes(quota);
double get usageRatio => quota <= 0 ? 0 : (usedSpace / quota).clamp(0.0, 1.0);
bool isActive(AiModelInfo model) {
final current = active[model.kind.toLowerCase()];
return current != null && current == model.id;
}
factory ModelsSnapshot.fromJson(Map<String, dynamic> json) {
final rawModels = json['models'];
final models = <AiModelInfo>[];
if (rawModels is List) {
for (final item in rawModels) {
if (item is Map<String, dynamic>) {
models.add(AiModelInfo.fromJson(item));
} else if (item is Map) {
models.add(AiModelInfo.fromJson(Map<String, dynamic>.from(item)));
}
}
}
final active = <String, String>{};
final rawActive = json['active'];
if (rawActive is Map) {
rawActive.forEach((key, value) {
active[key.toString().toLowerCase()] = value.toString();
});
}
return ModelsSnapshot(
models: models,
active: active,
usedSpace: AiModelInfo._asInt(json['used_space']),
quota: AiModelInfo._asInt(json['quota']),
);
}
}

View File

@@ -0,0 +1,92 @@
class CharacterPack {
const CharacterPack({
required this.filename,
required this.name,
required this.characterType,
required this.renderType,
this.live2dModel,
});
final String filename;
final String name;
final String characterType;
final String renderType;
final String? live2dModel;
bool get isLive2d => renderType.toLowerCase() == 'live2d';
String get typeLabel {
switch (characterType.toLowerCase()) {
case 'human':
return '数字人';
case 'singer':
return '歌姬';
case 'live2d':
return 'Live2D';
case 'pet':
default:
return '宠物';
}
}
IconDataLike get iconHint {
switch (characterType.toLowerCase()) {
case 'human':
return IconDataLike.person;
case 'singer':
return IconDataLike.mic;
case 'live2d':
return IconDataLike.sparkle;
default:
return IconDataLike.pets;
}
}
factory CharacterPack.fromJson(Map<String, dynamic> json) {
final character = json['character'];
final Map<String, dynamic> charMap = character is Map
? Map<String, dynamic>.from(character)
: const <String, dynamic>{};
final filename = json['filename']?.toString() ?? '';
return CharacterPack(
filename: filename,
name: charMap['name']?.toString().isNotEmpty == true
? charMap['name'].toString()
: filename,
characterType: charMap['character_type']?.toString() ?? 'pet',
renderType: charMap['render_type']?.toString() ?? 'video',
live2dModel: charMap['live2d_model']?.toString(),
);
}
}
/// Avoid importing Flutter material in pure model unit tests.
enum IconDataLike { pets, person, mic, sparkle }
class AvailableConfigs {
const AvailableConfigs({
required this.configs,
required this.active,
});
final List<CharacterPack> configs;
final String active;
factory AvailableConfigs.fromJson(Map<String, dynamic> json) {
final raw = json['configs'];
final list = <CharacterPack>[];
if (raw is List) {
for (final item in raw) {
if (item is Map<String, dynamic>) {
list.add(CharacterPack.fromJson(item));
} else if (item is Map) {
list.add(CharacterPack.fromJson(Map<String, dynamic>.from(item)));
}
}
}
return AvailableConfigs(
configs: list,
active: json['active']?.toString() ?? '',
);
}
}

View File

@@ -0,0 +1,45 @@
class ChatTurnResponse {
const ChatTurnResponse({
required this.sessionId,
required this.replyText,
this.transcription,
this.replyAudioPath,
this.error,
});
final String sessionId;
final String replyText;
final String? transcription;
final String? replyAudioPath;
final String? error;
bool get isOk => error == null || error!.isEmpty;
factory ChatTurnResponse.fromJson(Map<String, dynamic> json) {
return ChatTurnResponse(
sessionId: json['session_id']?.toString() ?? '',
replyText: json['reply_text']?.toString() ?? '',
transcription: json['transcription']?.toString(),
replyAudioPath: json['reply_audio_path']?.toString(),
error: json['error']?.toString(),
);
}
}
enum ChatBubbleRole { user, assistant, system }
class ChatBubble {
ChatBubble({
required this.role,
required this.text,
this.audioPath,
this.isPending = false,
DateTime? createdAt,
}) : createdAt = createdAt ?? DateTime.now();
final ChatBubbleRole role;
String text;
String? audioPath;
bool isPending;
final DateTime createdAt;
}

View File

@@ -0,0 +1,83 @@
import 'package:flutter/foundation.dart';
import '../models/character_pack.dart';
import '../services/http_api_service.dart';
import 'debug_provider.dart';
class CharacterProvider extends ChangeNotifier {
CharacterProvider({
required HttpApiService httpApiService,
required DebugProvider debugProvider,
}) : _http = httpApiService,
_debug = debugProvider;
final HttpApiService _http;
final DebugProvider _debug;
List<CharacterPack> _packs = const [];
String _active = '';
bool _loading = false;
bool _switching = false;
String? _error;
List<CharacterPack> get packs => _packs;
String get activeFilename => _active;
bool get isLoading => _loading;
bool get isSwitching => _switching;
String? get error => _error;
CharacterPack? get activePack {
for (final pack in _packs) {
if (pack.filename == _active) return pack;
}
return null;
}
Future<void> refresh() async {
_loading = true;
_error = null;
notifyListeners();
try {
final snapshot = await _http.getAvailableCharacterConfigs();
_packs = snapshot.configs;
_active = snapshot.active;
_debug.addHttpLog(
'Loaded characters',
details: 'count=${_packs.length} active=$_active',
);
} catch (error) {
_error = error.toString();
_debug.addHttpLog('Load characters failed', details: _error);
} finally {
_loading = false;
notifyListeners();
}
}
Future<bool> switchTo(String filename) async {
if (_switching || filename == _active) return false;
_switching = true;
_error = null;
notifyListeners();
try {
final result = await _http.switchConfig(filename);
if (!result.isOk) {
_error = result.message;
_debug.addHttpLog('Switch character failed', details: result.message);
return false;
}
_active = filename;
_debug.addHttpLog('Switched character', details: filename);
// 热重载后刷新列表,确保 active 与服务端一致
await refresh();
return true;
} catch (error) {
_error = error.toString();
_debug.addHttpLog('Switch character failed', details: _error);
return false;
} finally {
_switching = false;
notifyListeners();
}
}
}

View File

@@ -0,0 +1,270 @@
import 'dart:async';
import 'dart:io';
import 'package:audioplayers/audioplayers.dart';
import 'package:flutter/foundation.dart';
import 'package:path_provider/path_provider.dart';
import 'package:record/record.dart';
import '../models/chat_models.dart';
import '../services/http_api_service.dart';
import 'debug_provider.dart';
class ChatProvider extends ChangeNotifier {
ChatProvider({
required HttpApiService httpApiService,
required DebugProvider debugProvider,
}) : _http = httpApiService,
_debug = debugProvider;
final HttpApiService _http;
final DebugProvider _debug;
final AudioRecorder _recorder = AudioRecorder();
final AudioPlayer _player = AudioPlayer();
final List<ChatBubble> _messages = <ChatBubble>[];
String _sessionId = 'app_${DateTime.now().millisecondsSinceEpoch}';
bool _busy = false;
bool _recording = false;
String? _error;
String? _status;
List<ChatBubble> get messages => List<ChatBubble>.unmodifiable(_messages);
bool get isBusy => _busy;
bool get isRecording => _recording;
String? get error => _error;
String? get status => _status;
String get sessionId => _sessionId;
void clearHistory() {
_messages.clear();
_sessionId = 'app_${DateTime.now().millisecondsSinceEpoch}';
_error = null;
_status = '已清空对话';
notifyListeners();
}
Future<void> sendText(String text) async {
final content = text.trim();
if (content.isEmpty || _busy) return;
_messages.add(ChatBubble(role: ChatBubbleRole.user, text: content));
final pending = ChatBubble(
role: ChatBubbleRole.assistant,
text: '思考中...',
isPending: true,
);
_messages.add(pending);
_busy = true;
_error = null;
_status = '角色思考中...';
notifyListeners();
try {
final response = await _http.chatText(
text: content,
sessionId: _sessionId,
);
await _applyResponse(pending, response);
} catch (error) {
pending
..isPending = false
..text = '请求失败: $error';
_error = error.toString();
_debug.addHttpLog('chatText failed', details: _error);
} finally {
_busy = false;
if (_status == '角色思考中...') {
_status = null;
}
notifyListeners();
}
}
Future<bool> startRecording() async {
if (_busy || _recording) return false;
try {
final hasPermission = await _recorder.hasPermission();
if (!hasPermission) {
_error = '需要麦克风权限才能语音对话';
_status = _error;
notifyListeners();
return false;
}
final dir = await getTemporaryDirectory();
final path =
'${dir.path}/showen_rec_${DateTime.now().millisecondsSinceEpoch}.wav';
await _recorder.start(
const RecordConfig(
encoder: AudioEncoder.wav,
sampleRate: 16000,
numChannels: 1,
),
path: path,
);
_recording = true;
_error = null;
_status = '正在录音... 松开发送';
_debug.addBleLog('Voice recording started');
notifyListeners();
return true;
} catch (error) {
_error = '无法开始录音: $error';
_status = _error;
_debug.addHttpLog('startRecording failed', details: _error);
notifyListeners();
return false;
}
}
Future<void> cancelRecording() async {
if (!_recording) return;
try {
final path = await _recorder.stop();
if (path != null) {
final file = File(path);
if (await file.exists()) {
await file.delete();
}
}
} catch (_) {}
_recording = false;
_status = '已取消录音';
notifyListeners();
}
Future<void> stopRecordingAndSend() async {
if (!_recording || _busy) return;
_recording = false;
_status = '正在识别...';
notifyListeners();
String? path;
try {
path = await _recorder.stop();
} catch (error) {
_error = '停止录音失败: $error';
_status = _error;
notifyListeners();
return;
}
if (path == null) {
_error = '没有录到音频';
_status = _error;
notifyListeners();
return;
}
final file = File(path);
if (!await file.exists()) {
_error = '录音文件不存在';
_status = _error;
notifyListeners();
return;
}
final bytes = await file.readAsBytes();
try {
await file.delete();
} catch (_) {}
if (bytes.length < 1000) {
_error = '录音太短,请按住再说一会儿';
_status = _error;
notifyListeners();
return;
}
_messages.add(
ChatBubble(role: ChatBubbleRole.user, text: '🎤 语音消息'),
);
final pending = ChatBubble(
role: ChatBubbleRole.assistant,
text: '思考中...',
isPending: true,
);
_messages.add(pending);
_busy = true;
_status = '角色思考中...';
notifyListeners();
try {
final response = await _http.chatAudio(
bytes: bytes,
format: 'wav',
sessionId: _sessionId,
);
if (response.transcription != null &&
response.transcription!.trim().isNotEmpty) {
// 回填用户气泡为转写文本
for (var i = _messages.length - 1; i >= 0; i--) {
if (_messages[i].role == ChatBubbleRole.user) {
_messages[i].text = response.transcription!;
break;
}
}
}
await _applyResponse(pending, response);
} catch (error) {
pending
..isPending = false
..text = '语音对话失败: $error';
_error = error.toString();
_debug.addHttpLog('chatAudio failed', details: _error);
} finally {
_busy = false;
if (_status == '角色思考中...') {
_status = null;
}
notifyListeners();
}
}
Future<void> _applyResponse(
ChatBubble pending,
ChatTurnResponse response,
) async {
pending.isPending = false;
if (!response.isOk) {
pending.text = response.error ?? '对话失败';
_error = pending.text;
_status = pending.text;
return;
}
if (response.sessionId.isNotEmpty) {
_sessionId = response.sessionId;
}
pending.text = response.replyText.isEmpty ? '(空回复)' : response.replyText;
pending.audioPath = response.replyAudioPath;
_status = '已回复';
_debug.addHttpLog(
'chat ok',
details: 'chars=${pending.text.length} audio=${pending.audioPath != null}',
);
if (pending.audioPath != null && pending.audioPath!.isNotEmpty) {
unawaited(playReply(pending.audioPath!));
}
}
Future<void> playReply(String absolutePath) async {
try {
final url = _http.chatAudioFileUrl(absolutePath);
await _player.stop();
await _player.play(UrlSource(url));
_status = '正在播放回复...';
notifyListeners();
} catch (error) {
_error = '播放失败: $error';
_debug.addHttpLog('playReply failed', details: _error);
notifyListeners();
}
}
@override
void dispose() {
unawaited(_recorder.dispose());
unawaited(_player.dispose());
super.dispose();
}
}

View File

@@ -0,0 +1,116 @@
import 'dart:async';
import 'package:flutter/foundation.dart';
import '../models/ai_model.dart';
import '../models/api_response.dart';
import '../services/http_api_service.dart';
import 'debug_provider.dart';
class ModelProvider extends ChangeNotifier {
ModelProvider({
required HttpApiService httpApiService,
required DebugProvider debugProvider,
}) : _http = httpApiService,
_debug = debugProvider;
final HttpApiService _http;
final DebugProvider _debug;
ModelsSnapshot? _snapshot;
bool _loading = false;
bool _acting = false;
String? _error;
String? _status;
Timer? _pollTimer;
ModelsSnapshot? get snapshot => _snapshot;
bool get isLoading => _loading;
bool get isActing => _acting;
String? get error => _error;
String? get status => _status;
Future<void> refresh() async {
_loading = true;
_error = null;
notifyListeners();
try {
_snapshot = await _http.getModels();
_debug.addHttpLog(
'Loaded models',
details: 'count=${_snapshot?.models.length ?? 0}',
);
} catch (error) {
_error = error.toString();
_debug.addHttpLog('Load models failed', details: _error);
} finally {
_loading = false;
notifyListeners();
}
}
Future<void> download(String modelId) async {
await _runAction(() => _http.downloadModel(modelId), '下载已启动: $modelId');
_startPolling();
}
Future<void> switchTo({required String modelId, required String kind}) async {
await _runAction(
() => _http.switchModel(modelId: modelId, kind: kind),
'已切换: $modelId',
);
await refresh();
}
Future<void> delete(String modelId) async {
await _runAction(() => _http.deleteModel(modelId), '已删除: $modelId');
await refresh();
}
Future<void> _runAction(
Future<ApiResponse> Function() action,
String okStatus,
) async {
if (_acting) return;
_acting = true;
_error = null;
notifyListeners();
try {
final result = await action();
_status = result.message.isNotEmpty ? result.message : okStatus;
if (!result.isOk) {
_error = _status;
}
_debug.addHttpLog('Model action ok', details: _status);
} catch (error) {
_error = error.toString();
_status = _error;
_debug.addHttpLog('Model action failed', details: _error);
} finally {
_acting = false;
notifyListeners();
}
}
void _startPolling() {
_pollTimer?.cancel();
var ticks = 0;
_pollTimer = Timer.periodic(const Duration(seconds: 2), (timer) async {
ticks += 1;
await refresh();
final downloading = _snapshot?.models.any(
(m) => m.downloadProgress > 0 && m.downloadProgress < 100 && !m.downloaded,
) ??
false;
if (!downloading || ticks > 90) {
timer.cancel();
}
});
}
@override
void dispose() {
_pollTimer?.cancel();
super.dispose();
}
}

View File

@@ -31,16 +31,21 @@ class AppShell extends StatelessWidget {
selectedIcon: Icon(Icons.home),
label: '首页',
),
NavigationDestination(
icon: Icon(Icons.face_outlined),
selectedIcon: Icon(Icons.face),
label: '角色',
),
NavigationDestination(
icon: Icon(Icons.chat_bubble_outline),
selectedIcon: Icon(Icons.chat_bubble),
label: '对话',
),
NavigationDestination(
icon: Icon(Icons.play_circle_outline),
selectedIcon: Icon(Icons.play_circle),
label: '播放',
),
NavigationDestination(
icon: Icon(Icons.bolt_outlined),
selectedIcon: Icon(Icons.bolt),
label: '触发',
),
NavigationDestination(
icon: Icon(Icons.wifi_outlined),
selectedIcon: Icon(Icons.wifi),
@@ -51,11 +56,6 @@ class AppShell extends StatelessWidget {
selectedIcon: Icon(Icons.settings),
label: '设置',
),
NavigationDestination(
icon: Icon(Icons.bug_report_outlined),
selectedIcon: Icon(Icons.bug_report),
label: '调试',
),
],
),
);

View File

@@ -0,0 +1,202 @@
import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import '../models/character_pack.dart';
import '../providers/character_provider.dart';
import '../providers/chat_provider.dart';
import '../theme/app_colors.dart';
class CharactersScreen extends StatefulWidget {
const CharactersScreen({super.key});
@override
State<CharactersScreen> createState() => _CharactersScreenState();
}
class _CharactersScreenState extends State<CharactersScreen> {
@override
void initState() {
super.initState();
WidgetsBinding.instance.addPostFrameCallback((_) {
context.read<CharacterProvider>().refresh();
});
}
IconData _iconFor(CharacterPack pack) {
switch (pack.iconHint) {
case IconDataLike.person:
return Icons.person_rounded;
case IconDataLike.mic:
return Icons.mic_rounded;
case IconDataLike.sparkle:
return Icons.auto_awesome_rounded;
case IconDataLike.pets:
return Icons.pets_rounded;
}
}
@override
Widget build(BuildContext context) {
final provider = context.watch<CharacterProvider>();
return Scaffold(
appBar: AppBar(
title: const Text('角色'),
actions: [
IconButton(
onPressed: provider.isLoading ? null : () => provider.refresh(),
icon: const Icon(Icons.refresh_rounded),
),
],
),
body: RefreshIndicator(
onRefresh: provider.refresh,
child: ListView(
padding: const EdgeInsets.all(AppSpacing.md),
children: [
if (provider.error != null)
Padding(
padding: const EdgeInsets.only(bottom: AppSpacing.md),
child: Text(
provider.error!,
style: const TextStyle(color: AppColors.error),
),
),
Text(
'选择角色内容包,设备画面与对话人设会同步切换。',
style: Theme.of(context).textTheme.bodyMedium?.copyWith(
color: AppColors.textSecondary,
),
),
const SizedBox(height: AppSpacing.md),
if (provider.isLoading && provider.packs.isEmpty)
const Padding(
padding: EdgeInsets.only(top: 48),
child: Center(child: CircularProgressIndicator()),
)
else if (provider.packs.isEmpty)
const Padding(
padding: EdgeInsets.only(top: 48),
child: Center(child: Text('暂无角色配置包')),
)
else
...provider.packs.map((pack) {
final active = pack.filename == provider.activeFilename;
return Card(
margin: const EdgeInsets.only(bottom: AppSpacing.md),
child: InkWell(
borderRadius: BorderRadius.circular(AppRadius.large),
onTap: provider.isSwitching || active
? null
: () async {
final ok = await provider.switchTo(pack.filename);
if (!context.mounted) return;
if (ok) {
// 切换角色清空对话上下文PRD §2.4
context.read<ChatProvider>().clearHistory();
}
ScaffoldMessenger.of(context).showSnackBar(
SnackBar(
content: Text(
ok
? '已切换到 ${pack.name}'
: (provider.error ?? '切换失败'),
),
),
);
},
child: Padding(
padding: const EdgeInsets.all(AppSpacing.md),
child: Row(
children: [
CircleAvatar(
radius: 28,
backgroundColor: active
? AppColors.primary.withValues(alpha: 0.2)
: AppColors.background,
child: Icon(
_iconFor(pack),
color: active
? AppColors.primary
: AppColors.textSecondary,
),
),
const SizedBox(width: AppSpacing.md),
Expanded(
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Row(
children: [
Flexible(
child: Text(
pack.name,
style: Theme.of(context)
.textTheme
.titleMedium,
),
),
if (active) ...[
const SizedBox(width: 8),
Container(
padding: const EdgeInsets.symmetric(
horizontal: 8,
vertical: 2,
),
decoration: BoxDecoration(
color: AppColors.success
.withValues(alpha: 0.15),
borderRadius:
BorderRadius.circular(999),
),
child: const Text(
'使用中',
style: TextStyle(
color: AppColors.success,
fontSize: 12,
),
),
),
],
],
),
const SizedBox(height: 4),
Text(
'${pack.typeLabel} · ${pack.renderType} · ${pack.filename}',
style: Theme.of(context)
.textTheme
.bodySmall
?.copyWith(
color: AppColors.textSecondary,
),
),
],
),
),
if (provider.isSwitching && !active)
const SizedBox(
width: 20,
height: 20,
child: CircularProgressIndicator(strokeWidth: 2),
)
else
Icon(
active
? Icons.check_circle_rounded
: Icons.chevron_right_rounded,
color: active
? AppColors.success
: AppColors.textSecondary,
),
],
),
),
),
);
}),
],
),
),
);
}
}

View File

@@ -0,0 +1,232 @@
import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import '../models/chat_models.dart';
import '../providers/character_provider.dart';
import '../providers/chat_provider.dart';
import '../theme/app_colors.dart';
class ChatScreen extends StatefulWidget {
const ChatScreen({super.key});
@override
State<ChatScreen> createState() => _ChatScreenState();
}
class _ChatScreenState extends State<ChatScreen> {
final TextEditingController _controller = TextEditingController();
final ScrollController _scrollController = ScrollController();
@override
void initState() {
super.initState();
WidgetsBinding.instance.addPostFrameCallback((_) {
context.read<CharacterProvider>().refresh();
});
}
@override
void dispose() {
_controller.dispose();
_scrollController.dispose();
super.dispose();
}
void _scrollToEnd() {
WidgetsBinding.instance.addPostFrameCallback((_) {
if (!_scrollController.hasClients) return;
_scrollController.animateTo(
_scrollController.position.maxScrollExtent + 80,
duration: const Duration(milliseconds: 250),
curve: Curves.easeOut,
);
});
}
Future<void> _sendText() async {
final text = _controller.text;
_controller.clear();
await context.read<ChatProvider>().sendText(text);
_scrollToEnd();
}
@override
Widget build(BuildContext context) {
final chat = context.watch<ChatProvider>();
final character = context.watch<CharacterProvider>();
final activeName = character.activePack?.name ?? '当前角色';
return Scaffold(
appBar: AppBar(
title: Text('对话 · $activeName'),
actions: [
IconButton(
tooltip: '清空会话',
onPressed: chat.isBusy ? null : chat.clearHistory,
icon: const Icon(Icons.delete_outline_rounded),
),
],
),
body: Column(
children: [
if (chat.status != null)
Container(
width: double.infinity,
color: AppColors.card,
padding: const EdgeInsets.symmetric(
horizontal: AppSpacing.md,
vertical: 8,
),
child: Text(
chat.status!,
style: TextStyle(
color: chat.error != null
? AppColors.error
: AppColors.textSecondary,
),
),
),
Expanded(
child: chat.messages.isEmpty
? Center(
child: Padding(
padding: const EdgeInsets.all(AppSpacing.lg),
child: Text(
'文字发送,或按住下方麦克风说话。\n回复会在手机播放(设备本地不出声)。',
textAlign: TextAlign.center,
style: Theme.of(context).textTheme.bodyMedium?.copyWith(
color: AppColors.textSecondary,
),
),
),
)
: ListView.builder(
controller: _scrollController,
padding: const EdgeInsets.all(AppSpacing.md),
itemCount: chat.messages.length,
itemBuilder: (context, index) {
final msg = chat.messages[index];
final isUser = msg.role == ChatBubbleRole.user;
return Align(
alignment: isUser
? Alignment.centerRight
: Alignment.centerLeft,
child: Container(
margin: const EdgeInsets.only(bottom: 10),
padding: const EdgeInsets.symmetric(
horizontal: 14,
vertical: 10,
),
constraints: BoxConstraints(
maxWidth: MediaQuery.of(context).size.width * 0.82,
),
decoration: BoxDecoration(
color: isUser
? AppColors.primary.withValues(alpha: 0.25)
: AppColors.card,
borderRadius: BorderRadius.circular(16),
border: Border.all(color: AppColors.border),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
msg.text,
style: Theme.of(context).textTheme.bodyMedium,
),
if (msg.audioPath != null &&
msg.audioPath!.isNotEmpty &&
!msg.isPending) ...[
const SizedBox(height: 8),
TextButton.icon(
onPressed: () =>
chat.playReply(msg.audioPath!),
icon: const Icon(Icons.volume_up_rounded),
label: const Text('重播语音'),
),
],
if (msg.isPending)
const Padding(
padding: EdgeInsets.only(top: 6),
child: SizedBox(
width: 14,
height: 14,
child: CircularProgressIndicator(
strokeWidth: 2,
),
),
),
],
),
),
);
},
),
),
SafeArea(
top: false,
child: Padding(
padding: const EdgeInsets.fromLTRB(12, 8, 12, 12),
child: Row(
children: [
Expanded(
child: TextField(
controller: _controller,
enabled: !chat.isBusy && !chat.isRecording,
minLines: 1,
maxLines: 4,
decoration: const InputDecoration(
hintText: '输入文字和角色对话...',
),
onSubmitted: (_) => _sendText(),
),
),
const SizedBox(width: 8),
IconButton.filled(
onPressed: chat.isBusy || chat.isRecording ? null : _sendText,
icon: const Icon(Icons.send_rounded),
),
const SizedBox(width: 4),
GestureDetector(
onLongPressStart: (_) async {
final ok =
await context.read<ChatProvider>().startRecording();
if (!ok && context.mounted) {
ScaffoldMessenger.of(context).showSnackBar(
SnackBar(
content: Text(
context.read<ChatProvider>().error ?? '无法录音',
),
),
);
}
},
onLongPressEnd: (_) async {
await context.read<ChatProvider>().stopRecordingAndSend();
_scrollToEnd();
},
onLongPressCancel: () {
context.read<ChatProvider>().cancelRecording();
},
child: CircleAvatar(
radius: 26,
backgroundColor: chat.isRecording
? AppColors.error
: AppColors.primary.withValues(alpha: 0.2),
child: Icon(
chat.isRecording ? Icons.mic : Icons.mic_none_rounded,
color: chat.isRecording
? Colors.white
: AppColors.primary,
),
),
),
],
),
),
),
],
),
);
}
}

View File

@@ -1,6 +1,8 @@
import 'package:flutter/material.dart';
import 'package:go_router/go_router.dart';
import 'package:provider/provider.dart';
import '../providers/character_provider.dart';
import '../providers/device_provider.dart';
import '../providers/player_provider.dart';
import '../providers/wifi_provider.dart';
@@ -16,9 +18,11 @@ class HomeScreen extends StatelessWidget {
final deviceProvider = context.watch<DeviceProvider>();
final playerProvider = context.watch<PlayerProvider>();
final wifiProvider = context.watch<WifiProvider>();
final characterProvider = context.watch<CharacterProvider>();
final device = deviceProvider.status;
final player = playerProvider.status;
final wifi = wifiProvider.status;
final characterName = characterProvider.activePack?.name ?? '未同步';
return Scaffold(
appBar: AppBar(title: const Text('ShowenV2 控制台')),
@@ -28,6 +32,7 @@ class HomeScreen extends StatelessWidget {
context.read<DeviceProvider>().refresh(),
context.read<PlayerProvider>().bootstrap(),
context.read<WifiProvider>().bootstrap(),
context.read<CharacterProvider>().refresh(),
]);
},
child: ListView(
@@ -41,6 +46,16 @@ class HomeScreen extends StatelessWidget {
accentColor: device.connected ? AppColors.success : AppColors.warning,
),
const SizedBox(height: AppSpacing.md),
StatusCard(
title: '当前角色',
value: characterName,
subtitle: characterProvider.activeFilename.isEmpty
? '点击角色页切换内容包'
: characterProvider.activeFilename,
icon: Icons.face_rounded,
accentColor: AppColors.secondary,
),
const SizedBox(height: AppSpacing.md),
StatusCard(
title: '当前播放状态',
value: player.currentVideo ?? '暂无播放视频',
@@ -59,6 +74,27 @@ class HomeScreen extends StatelessWidget {
accentColor: wifi.connected ? AppColors.info : AppColors.border,
),
const SizedBox(height: AppSpacing.lg),
Row(
children: [
Expanded(
child: ControlButton(
label: '去对话',
icon: Icons.chat_bubble_rounded,
onPressed: () => context.go('/chat'),
),
),
const SizedBox(width: AppSpacing.md),
Expanded(
child: ControlButton(
label: '换角色',
icon: Icons.face_rounded,
isFilled: false,
onPressed: () => context.go('/characters'),
),
),
],
),
const SizedBox(height: AppSpacing.lg),
Text('快捷控制', style: Theme.of(context).textTheme.headlineSmall),
const SizedBox(height: AppSpacing.md),
Row(

View File

@@ -0,0 +1,229 @@
import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import '../models/ai_model.dart';
import '../providers/model_provider.dart';
import '../theme/app_colors.dart';
class ModelsScreen extends StatefulWidget {
const ModelsScreen({super.key});
@override
State<ModelsScreen> createState() => _ModelsScreenState();
}
class _ModelsScreenState extends State<ModelsScreen> {
@override
void initState() {
super.initState();
WidgetsBinding.instance.addPostFrameCallback((_) {
context.read<ModelProvider>().refresh();
});
}
@override
Widget build(BuildContext context) {
final provider = context.watch<ModelProvider>();
final snapshot = provider.snapshot;
return Scaffold(
appBar: AppBar(
title: const Text('模型管理'),
actions: [
IconButton(
onPressed: provider.isLoading ? null : () => provider.refresh(),
icon: const Icon(Icons.refresh_rounded),
),
],
),
body: RefreshIndicator(
onRefresh: provider.refresh,
child: ListView(
padding: const EdgeInsets.all(AppSpacing.md),
children: [
if (provider.error != null)
Padding(
padding: const EdgeInsets.only(bottom: AppSpacing.sm),
child: Text(
provider.error!,
style: const TextStyle(color: AppColors.error),
),
),
if (provider.status != null)
Padding(
padding: const EdgeInsets.only(bottom: AppSpacing.sm),
child: Text(
provider.status!,
style: const TextStyle(color: AppColors.textSecondary),
),
),
if (snapshot != null) ...[
Card(
child: Padding(
padding: const EdgeInsets.all(AppSpacing.md),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
'存储 ${snapshot.usedLabel} / ${snapshot.quotaLabel}',
style: Theme.of(context).textTheme.titleMedium,
),
const SizedBox(height: 8),
LinearProgressIndicator(value: snapshot.usageRatio),
const SizedBox(height: 8),
Text(
'激活: LLM=${snapshot.active['llm'] ?? '-'} · '
'ASR=${snapshot.active['asr'] ?? '-'} · '
'TTS=${snapshot.active['tts'] ?? '-'}',
style: Theme.of(context).textTheme.bodySmall?.copyWith(
color: AppColors.textSecondary,
),
),
],
),
),
),
const SizedBox(height: AppSpacing.md),
],
if (provider.isLoading && snapshot == null)
const Padding(
padding: EdgeInsets.only(top: 48),
child: Center(child: CircularProgressIndicator()),
)
else if (snapshot == null || snapshot.models.isEmpty)
const Padding(
padding: EdgeInsets.only(top: 48),
child: Center(child: Text('暂无模型清单AI 插件可能未就绪)')),
)
else
...snapshot.models.map((model) {
final active = snapshot.isActive(model);
return Card(
margin: const EdgeInsets.only(bottom: AppSpacing.md),
child: Padding(
padding: const EdgeInsets.all(AppSpacing.md),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Row(
children: [
Expanded(
child: Text(
model.name,
style: Theme.of(context).textTheme.titleMedium,
),
),
if (active)
Container(
padding: const EdgeInsets.symmetric(
horizontal: 8,
vertical: 2,
),
decoration: BoxDecoration(
color:
AppColors.success.withValues(alpha: 0.15),
borderRadius: BorderRadius.circular(999),
),
child: const Text(
'使用中',
style: TextStyle(
color: AppColors.success,
fontSize: 12,
),
),
),
],
),
const SizedBox(height: 4),
Text(
'${model.kindLabel} · ${model.sizeLabel} · 内存 ${model.memoryLabel}',
style:
Theme.of(context).textTheme.bodySmall?.copyWith(
color: AppColors.textSecondary,
),
),
if (model.recommendedTier.isNotEmpty)
Text(
model.recommendedTier,
style: Theme.of(context)
.textTheme
.bodySmall
?.copyWith(color: AppColors.textSecondary),
),
if (!model.downloaded && model.downloadProgress > 0)
Padding(
padding: const EdgeInsets.only(top: 8),
child: LinearProgressIndicator(
value: model.downloadProgress / 100.0,
),
),
const SizedBox(height: 10),
Wrap(
spacing: 8,
runSpacing: 8,
children: [
if (!model.downloaded)
FilledButton.tonal(
onPressed: provider.isActing
? null
: () => provider.download(model.id),
child: Text(
model.downloadProgress > 0 &&
model.downloadProgress < 100
? '下载中 ${model.downloadProgress}%'
: '下载',
),
),
if (model.downloaded && !active)
FilledButton(
onPressed: provider.isActing
? null
: () => provider.switchTo(
modelId: model.id,
kind: model.kind,
),
child: const Text('切换使用'),
),
if (model.downloaded && !active)
OutlinedButton(
onPressed: provider.isActing
? null
: () => _confirmDelete(context, model),
child: const Text('删除'),
),
],
),
],
),
),
);
}),
],
),
),
);
}
Future<void> _confirmDelete(BuildContext context, AiModelInfo model) async {
final ok = await showDialog<bool>(
context: context,
builder: (context) => AlertDialog(
title: const Text('删除模型'),
content: Text('确定删除 ${model.name}?此操作不可恢复。'),
actions: [
TextButton(
onPressed: () => Navigator.pop(context, false),
child: const Text('取消'),
),
FilledButton(
onPressed: () => Navigator.pop(context, true),
child: const Text('删除'),
),
],
),
);
if (ok == true && context.mounted) {
await context.read<ModelProvider>().delete(model.id);
}
}
}

View File

@@ -2,6 +2,7 @@ import 'dart:convert';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:go_router/go_router.dart';
import 'package:provider/provider.dart';
import '../models/video_item.dart';
@@ -80,6 +81,34 @@ class _SettingsScreenState extends State<SettingsScreen> {
physics: const AlwaysScrollableScrollPhysics(),
padding: const EdgeInsets.all(AppSpacing.md),
children: [
Card(
child: Column(
children: [
ListTile(
leading: const Icon(Icons.memory_rounded),
title: const Text('模型管理'),
subtitle: const Text('下载 / 切换 / 删除 AI 模型'),
trailing: const Icon(Icons.chevron_right_rounded),
onTap: () => context.push('/settings/models'),
),
const Divider(height: 1),
ListTile(
leading: const Icon(Icons.bolt_rounded),
title: const Text('状态机触发'),
trailing: const Icon(Icons.chevron_right_rounded),
onTap: () => context.push('/settings/trigger'),
),
const Divider(height: 1),
ListTile(
leading: const Icon(Icons.bug_report_rounded),
title: const Text('调试日志'),
trailing: const Icon(Icons.chevron_right_rounded),
onTap: () => context.push('/settings/debug'),
),
],
),
),
const SizedBox(height: AppSpacing.lg),
Card(
child: Padding(
padding: const EdgeInsets.all(AppSpacing.md),
@@ -464,9 +493,17 @@ class _SettingsScreenState extends State<SettingsScreen> {
Map<String, dynamic>.from(results[0] as Map<String, dynamic>);
final available =
Map<String, dynamic>.from(results[1] as Map<String, dynamic>);
// M2.1: configs 为 {filename, character} 对象列表;兼容旧版纯字符串列表
_availableConfigs =
(available['configs'] as List<dynamic>? ?? const <dynamic>[])
.map((item) => item.toString())
.map((item) {
if (item is String) return item;
if (item is Map) {
return item['filename']?.toString() ?? item.toString();
}
return item.toString();
})
.where((name) => name.isNotEmpty)
.toList(growable: false);
_activeConfig = available['active']?.toString();
_applyDisplayConfig(Map<String, dynamic>.from(

View File

@@ -5,8 +5,11 @@ import 'dart:typed_data';
import 'package:http/http.dart' as http;
import '../models/ai_model.dart';
import '../models/api_response.dart';
import '../models/ble_status.dart';
import '../models/character_pack.dart';
import '../models/chat_models.dart';
import '../models/player_status.dart';
import '../models/video_item.dart';
import '../models/wifi_network.dart';
@@ -300,6 +303,118 @@ class HttpApiService {
return ApiResponse.fromJson(_decodeMap(response.body));
}
Future<AvailableConfigs> getAvailableCharacterConfigs() async {
final map = await getAvailableConfigs();
return AvailableConfigs.fromJson(map);
}
// ── M2.1 AI 对话 / 模型管理 ──
Future<ChatTurnResponse> chatText({
required String text,
String? sessionId,
}) async {
final response = await _client.post(
_uri('/api/chat/text'),
headers: _jsonHeaders,
body: jsonEncode(<String, dynamic>{
'text': text,
if (sessionId != null && sessionId.isNotEmpty) 'session_id': sessionId,
}),
);
// AI 错误也以 JSON ChatResponse 返回(可能是 5xx
final map = _decodeMap(response.body);
final result = ChatTurnResponse.fromJson(map);
if (response.statusCode >= 200 && response.statusCode < 300) {
return result;
}
if (result.error != null && result.error!.isNotEmpty) {
return result;
}
throw ApiException(
map['message']?.toString() ?? '对话失败 (${response.statusCode})',
statusCode: response.statusCode,
);
}
Future<ChatTurnResponse> chatAudio({
required List<int> bytes,
String format = 'wav',
String? sessionId,
}) async {
final response = await _client.post(
_uri('/api/chat/audio'),
headers: <String, String>{
'Content-Type': 'application/octet-stream',
'Accept': 'application/json',
'X-Audio-Format': format,
if (sessionId != null && sessionId.isNotEmpty) 'X-Session-Id': sessionId,
},
body: bytes,
);
final map = _decodeMap(response.body);
final result = ChatTurnResponse.fromJson(map);
if (response.statusCode >= 200 && response.statusCode < 300) {
return result;
}
if (result.error != null && result.error!.isNotEmpty) {
return result;
}
throw ApiException(
map['message']?.toString() ?? '语音对话失败 (${response.statusCode})',
statusCode: response.statusCode,
);
}
String chatAudioFileUrl(String absolutePath) {
return _uri(
'/api/chat/audio/file',
<String, String>{'path': absolutePath},
).toString();
}
Future<ModelsSnapshot> getModels() async {
final response = await _client.get(_uri('/api/models'));
_ensureSuccess(response);
return ModelsSnapshot.fromJson(_decodeMap(response.body));
}
Future<ApiResponse> downloadModel(String modelId) async {
final response = await _client.post(
_uri('/api/models/download'),
headers: _jsonHeaders,
body: jsonEncode(<String, dynamic>{'model_id': modelId}),
);
_ensureSuccess(response);
return ApiResponse.fromJson(_decodeMap(response.body));
}
Future<ApiResponse> switchModel({
required String modelId,
required String kind,
}) async {
final response = await _client.post(
_uri('/api/models/switch'),
headers: _jsonHeaders,
body: jsonEncode(<String, dynamic>{
'model_id': modelId,
'kind': kind,
}),
);
_ensureSuccess(response);
return ApiResponse.fromJson(_decodeMap(response.body));
}
Future<ApiResponse> deleteModel(String modelId) async {
final response = await _client.post(
_uri('/api/models/delete'),
headers: _jsonHeaders,
body: jsonEncode(<String, dynamic>{'model_id': modelId}),
);
_ensureSuccess(response);
return ApiResponse.fromJson(_decodeMap(response.body));
}
Future<List<Map<String, dynamic>>> listFiles(String dirKey, [String? path]) async {
final response = await _client.get(
_uri('/api/files/$dirKey', _pathQuery(path)),

View File

@@ -6,23 +6,79 @@ packages:
description:
name: args
sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.7.0"
async:
dependency: transitive
description:
name: async
sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb"
url: "https://pub.dev"
sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.13.0"
version: "2.13.1"
audioplayers:
dependency: "direct main"
description:
name: audioplayers
sha256: "2ba4bb2944baacbdd5372ff8254a8e7feb8c10d7739545e392f5605a8f618745"
url: "https://pub.flutter-io.cn"
source: hosted
version: "6.8.1"
audioplayers_android:
dependency: transitive
description:
name: audioplayers_android
sha256: f5ff5b15620fbab8cb0849e9636c48e2b96c3f0f71723bbbe2ad3c761b205f05
url: "https://pub.flutter-io.cn"
source: hosted
version: "5.3.0"
audioplayers_darwin:
dependency: transitive
description:
name: audioplayers_darwin
sha256: "1ca553add991384ecf421b9569da850f3ab2472ffb83f6970b0416365abc51be"
url: "https://pub.flutter-io.cn"
source: hosted
version: "6.5.0"
audioplayers_linux:
dependency: transitive
description:
name: audioplayers_linux
sha256: "15178b726b7cdee5364d0463c8d445630c4e0fb7d26612b73c767e7d25de9417"
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.3.0"
audioplayers_platform_interface:
dependency: transitive
description:
name: audioplayers_platform_interface
sha256: "765f6f0e6dca55cb471c9483fc77700564b3484d19198aca4ebb5147c6c85acb"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.2.0"
audioplayers_web:
dependency: transitive
description:
name: audioplayers_web
sha256: ae1e0103c865a03e273f6d13d97b93f5595eac09915729cd5e37ef96e2857319
url: "https://pub.flutter-io.cn"
source: hosted
version: "5.3.0"
audioplayers_windows:
dependency: transitive
description:
name: audioplayers_windows
sha256: a70ae82bba2dfcb6eb03dd4815d737a2d46d33ea5a96a03f535cfcaac490e413
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.4.1"
bluez:
dependency: transitive
description:
name: bluez
sha256: "61a7204381925896a374301498f2f5399e59827c6498ae1e924aaa598751b545"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.8.3"
boolean_selector:
@@ -30,7 +86,7 @@ packages:
description:
name: boolean_selector
sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.1.2"
characters:
@@ -38,7 +94,7 @@ packages:
description:
name: characters
sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.4.1"
clock:
@@ -46,15 +102,23 @@ packages:
description:
name: clock
sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.1.2"
code_assets:
dependency: transitive
description:
name: code_assets
sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.2.1"
collection:
dependency: transitive
description:
name: collection
sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.19.1"
crypto:
@@ -62,31 +126,31 @@ packages:
description:
name: crypto
sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.0.7"
cupertino_icons:
dependency: "direct main"
description:
name: cupertino_icons
sha256: ba631d1c7f7bef6b729a622b7b752645a2d076dba9976925b8f25725a30e1ee6
url: "https://pub.dev"
sha256: "41e005c33bd814be4d3096aff55b1908d419fde52ca656c8c47719ec745873cd"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.8"
version: "1.0.9"
dbus:
dependency: transitive
description:
name: dbus
sha256: d0c98dcd4f5169878b6cf8f6e0a52403a9dff371a3e2f019697accbf6f44a270
url: "https://pub.dev"
sha256: "0ce9b0a839e6dee59a37a623d2fc26a35bbbe6404213e419b0d6411023d62645"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.7.12"
version: "0.7.14"
fake_async:
dependency: transitive
description:
name: fake_async
sha256: "5368f224a74523e8d2e7399ea1638b37aecfca824a3cc4dfdf77bf1fa905ac44"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.3.3"
ffi:
@@ -94,7 +158,7 @@ packages:
description:
name: ffi
sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.2.0"
file:
@@ -102,9 +166,17 @@ packages:
description:
name: file
sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.1"
fixnum:
dependency: transitive
description:
name: fixnum
sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.1.1"
flutter:
dependency: "direct main"
description: flutter
@@ -115,7 +187,7 @@ packages:
description:
name: flutter_blue_plus
sha256: "69a8c87c11fc792e8cf0f997d275484fbdb5143ac9f0ac4d424429700cb4e0ed"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.36.8"
flutter_blue_plus_android:
@@ -123,7 +195,7 @@ packages:
description:
name: flutter_blue_plus_android
sha256: "6f7fe7e69659c30af164a53730707edc16aa4d959e4c208f547b893d940f853d"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.4"
flutter_blue_plus_darwin:
@@ -131,7 +203,7 @@ packages:
description:
name: flutter_blue_plus_darwin
sha256: "682982862c1d964f4d54a3fb5fccc9e59a066422b93b7e22079aeecd9c0d38f8"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.3"
flutter_blue_plus_linux:
@@ -139,7 +211,7 @@ packages:
description:
name: flutter_blue_plus_linux
sha256: "56b0c45edd0a2eec8f85bd97a26ac3cd09447e10d0094fed55587bf0592e3347"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.3"
flutter_blue_plus_platform_interface:
@@ -147,7 +219,7 @@ packages:
description:
name: flutter_blue_plus_platform_interface
sha256: "84fbd180c50a40c92482f273a92069960805ce324e3673ad29c41d2faaa7c5c2"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.0"
flutter_blue_plus_web:
@@ -155,7 +227,7 @@ packages:
description:
name: flutter_blue_plus_web
sha256: a1aceee753d171d24c0e0cdadb37895b5e9124862721f25f60bb758e20b72c99
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.2"
flutter_lints:
@@ -163,7 +235,7 @@ packages:
description:
name: flutter_lints
sha256: "3f41d009ba7172d5ff9be5f6e6e6abb4300e263aab8866d2a0842ed2a70f8f0c"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.0.0"
flutter_test:
@@ -181,15 +253,23 @@ packages:
description:
name: go_router
sha256: f02fd7d2a4dc512fec615529824fdd217fecb3a3d3de68360293a551f21634b3
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "14.8.1"
hooks:
dependency: transitive
description:
name: hooks
sha256: "9a62a50b50b769a737bc0a8ff381f333529df3ab746b2f6b02e83760231455ba"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.0.2"
http:
dependency: "direct main"
description:
name: http
sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.6.0"
http_parser:
@@ -197,15 +277,31 @@ packages:
description:
name: http_parser
sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.1.2"
jni:
dependency: transitive
description:
name: jni
sha256: c2230682d5bc2362c1c9e8d3c7f406d9cbba23ab3f2e203a025dd47e0fb2e68f
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.0"
jni_flutter:
dependency: transitive
description:
name: jni_flutter
sha256: "8b59e590786050b1cd866677dddaf76b1ade5e7bc751abe04b86e84d379d3ba6"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.1"
leak_tracker:
dependency: transitive
description:
name: leak_tracker
sha256: "33e2e26bdd85a0112ec15400c8cbffea70d0f9c3407491f672a2fad47915e2de"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "11.0.2"
leak_tracker_flutter_testing:
@@ -213,7 +309,7 @@ packages:
description:
name: leak_tracker_flutter_testing
sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.0.10"
leak_tracker_testing:
@@ -221,7 +317,7 @@ packages:
description:
name: leak_tracker_testing
sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.0.2"
lints:
@@ -229,7 +325,7 @@ packages:
description:
name: lints
sha256: "976c774dd944a42e83e2467f4cc670daef7eed6295b10b36ae8c85bcbf828235"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.0.0"
logging:
@@ -237,7 +333,7 @@ packages:
description:
name: logging
sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.3.0"
matcher:
@@ -245,7 +341,7 @@ packages:
description:
name: matcher
sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.12.19"
material_color_utilities:
@@ -253,55 +349,95 @@ packages:
description:
name: material_color_utilities
sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.13.0"
meta:
dependency: transitive
description:
name: meta
sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
url: "https://pub.dev"
sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.17.0"
version: "1.18.0"
nested:
dependency: transitive
description:
name: nested
sha256: "03bac4c528c64c95c722ec99280375a6f2fc708eec17c7b3f07253b626cd2a20"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.0"
objective_c:
dependency: transitive
description:
name: objective_c
sha256: "6cb691c686fa2838c6deb34980d426145c2a5d537491cb83d463c33cdbc726ed"
url: "https://pub.flutter-io.cn"
source: hosted
version: "9.4.1"
package_config:
dependency: transitive
description:
name: package_config
sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.2.0"
path:
dependency: transitive
description:
name: path
sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.9.1"
path_provider:
dependency: "direct main"
description:
name: path_provider
sha256: a7f4874f987173da295a61c181b8ee71dab59b332a486b391babf26a1b884825
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.1.6"
path_provider_android:
dependency: transitive
description:
name: path_provider_android
sha256: "69cbd515a62b94d32a7944f086b2f82b4ac40a1d45bebfc00813a430ab2dabcd"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.3.1"
path_provider_foundation:
dependency: transitive
description:
name: path_provider_foundation
sha256: "2a376b7d6392d80cd3705782d2caa734ca4727776db0b6ec36ef3f1855197699"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.6.0"
path_provider_linux:
dependency: transitive
description:
name: path_provider_linux
sha256: f7a1fe3a634fe7734c8d3f2766ad746ae2a2884abe22e241a8b301bf5cac3279
url: "https://pub.dev"
sha256: "58c2005f147315b11e9b4a7bc889cd5203e250cba8e3f012dae259b4972b5c16"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.2.1"
version: "2.2.2"
path_provider_platform_interface:
dependency: transitive
description:
name: path_provider_platform_interface
sha256: "88f5779f72ba699763fa3a3b06aa4bf6de76c8e5de842cf6f29e2e06476c2334"
url: "https://pub.dev"
sha256: "484838772624c3a4b94f1e44a3e19897fee738f2d5c4ce448443b0417f7c9dda"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.1.2"
version: "2.1.3"
path_provider_windows:
dependency: transitive
description:
name: path_provider_windows
sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.3.0"
petitparser:
@@ -309,7 +445,7 @@ packages:
description:
name: petitparser
sha256: "91bd59303e9f769f108f8df05e371341b15d59e995e6806aefab827b58336675"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "7.0.2"
platform:
@@ -317,7 +453,7 @@ packages:
description:
name: platform
sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.1.6"
plugin_platform_interface:
@@ -325,7 +461,7 @@ packages:
description:
name: plugin_platform_interface
sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.1.8"
provider:
@@ -333,39 +469,119 @@ packages:
description:
name: provider
sha256: "4e82183fa20e5ca25703ead7e05de9e4cceed1fbd1eadc1ac3cb6f565a09f272"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "6.1.5+1"
pub_semver:
dependency: transitive
description:
name: pub_semver
sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.2.0"
record:
dependency: "direct main"
description:
name: record
sha256: "10911465138fafacef459a780564e883e01bd48eabf87ab20543684884492870"
url: "https://pub.flutter-io.cn"
source: hosted
version: "6.2.1"
record_android:
dependency: transitive
description:
name: record_android
sha256: eb1732e42d0d2a1895b8db86e4fc917287e6d8491b6ed59918aea8bed6c69de4
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.5.2"
record_ios:
dependency: transitive
description:
name: record_ios
sha256: c051fb48edd7a0e265daafb9108730dc827c27b551728a3fdfb3ef69efd89c73
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.2.1"
record_linux:
dependency: transitive
description:
name: record_linux
sha256: "31181787bf7eccb0e298835836b69b3cd0a903863b75d70e937de3dec71cd8f3"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.3.1"
record_macos:
dependency: transitive
description:
name: record_macos
sha256: cfe1b61435e27db418bf513dc36820d10c9f7eb1843786c2c9a52e07e2f4f627
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.2.2"
record_platform_interface:
dependency: transitive
description:
name: record_platform_interface
sha256: "8e56cbe06c6984137fb86132ff03459f29938d927496d9b2d0962e2d6345d488"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.6.0"
record_use:
dependency: transitive
description:
name: record_use
sha256: "2551bd8eecfe95d14ae75f6021ad0248be5c27f138c2ec12fcb52b500b3ba1ed"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.6.0"
record_web:
dependency: transitive
description:
name: record_web
sha256: "7e9846981c1f2d111d86f0ae3309071f5bba8b624d1c977316706f08fc31d16d"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.3.0"
record_windows:
dependency: transitive
description:
name: record_windows
sha256: "223258060a1d25c62bae18282c16783f28581ec19401d17e56b5205b9f039d78"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.7"
rxdart:
dependency: transitive
description:
name: rxdart
sha256: "5c3004a4a8dbb94bd4bf5412a4def4acdaa12e12f269737a5751369e12d1a962"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.28.0"
shared_preferences:
dependency: "direct main"
description:
name: shared_preferences
sha256: "2939ae520c9024cb197fc20dee269cd8cdbf564c8b5746374ec6cacdc5169e64"
url: "https://pub.dev"
sha256: c3025c5534b01739267eb7d76959bbc25a6d10f6988e1c2a3036940133dd10bf
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.5.4"
version: "2.5.5"
shared_preferences_android:
dependency: transitive
description:
name: shared_preferences_android
sha256: "8374d6200ab33ac99031a852eba4c8eb2170c4bf20778b3e2c9eccb45384fb41"
url: "https://pub.dev"
sha256: "93ae5884a9df5d3bb696825bceb3a17590754548b5d740eba51500afc8d088f5"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.4.21"
version: "2.4.26"
shared_preferences_foundation:
dependency: transitive
description:
name: shared_preferences_foundation
sha256: "4e7eaffc2b17ba398759f1151415869a34771ba11ebbccd1b0145472a619a64f"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.5.6"
shared_preferences_linux:
@@ -373,23 +589,23 @@ packages:
description:
name: shared_preferences_linux
sha256: "580abfd40f415611503cae30adf626e6656dfb2f0cee8f465ece7b6defb40f2f"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.4.1"
shared_preferences_platform_interface:
dependency: transitive
description:
name: shared_preferences_platform_interface
sha256: "57cbf196c486bc2cf1f02b85784932c6094376284b3ad5779d1b1c6c6a816b80"
url: "https://pub.dev"
sha256: "649dc798a33931919ea356c4305c2d1f81619ea6e92244070b520187b5140ef9"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.4.1"
version: "2.4.2"
shared_preferences_web:
dependency: transitive
description:
name: shared_preferences_web
sha256: c49bd060261c9a3f0ff445892695d6212ff603ef3115edbb448509d407600019
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.4.3"
shared_preferences_windows:
@@ -397,7 +613,7 @@ packages:
description:
name: shared_preferences_windows
sha256: "94ef0f72b2d71bc3e700e025db3710911bd51a71cefb65cc609dd0d9a982e3c1"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.4.1"
sky_engine:
@@ -410,7 +626,7 @@ packages:
description:
name: source_span
sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.10.2"
stack_trace:
@@ -418,7 +634,7 @@ packages:
description:
name: stack_trace
sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.12.1"
stream_channel:
@@ -426,7 +642,7 @@ packages:
description:
name: stream_channel
sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.1.4"
string_scanner:
@@ -434,55 +650,71 @@ packages:
description:
name: string_scanner
sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.4.1"
synchronized:
dependency: transitive
description:
name: synchronized
sha256: "93b153dcb6a26dcddee6ca087dd634b53e38c10b5aa163e8e49501a776456153"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.4.1"
term_glyph:
dependency: transitive
description:
name: term_glyph
sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.2.2"
test_api:
dependency: transitive
description:
name: test_api
sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a"
url: "https://pub.dev"
sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e"
url: "https://pub.flutter-io.cn"
source: hosted
version: "0.7.10"
version: "0.7.11"
typed_data:
dependency: transitive
description:
name: typed_data
sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.4.0"
uuid:
dependency: transitive
description:
name: uuid
sha256: "1fef9e8e11e2991bb773070d4656b7bd5d850967a2456cfc83cf47925ba79489"
url: "https://pub.flutter-io.cn"
source: hosted
version: "4.5.3"
vector_math:
dependency: transitive
description:
name: vector_math
sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "2.2.0"
vm_service:
dependency: transitive
description:
name: vm_service
sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60"
url: "https://pub.dev"
sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360"
url: "https://pub.flutter-io.cn"
source: hosted
version: "15.0.2"
version: "15.2.0"
web:
dependency: transitive
description:
name: web
sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.1.1"
web_socket:
@@ -490,7 +722,7 @@ packages:
description:
name: web_socket
sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.0.1"
web_socket_channel:
@@ -498,7 +730,7 @@ packages:
description:
name: web_socket_channel
sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.0.3"
xdg_directories:
@@ -506,7 +738,7 @@ packages:
description:
name: xdg_directories
sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "1.1.0"
xml:
@@ -514,9 +746,17 @@ packages:
description:
name: xml
sha256: "971043b3a0d3da28727e40ed3e0b5d18b742fa5a68665cca88e74b7876d5e025"
url: "https://pub.dev"
url: "https://pub.flutter-io.cn"
source: hosted
version: "6.6.1"
yaml:
dependency: transitive
description:
name: yaml
sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce
url: "https://pub.flutter-io.cn"
source: hosted
version: "3.1.3"
sdks:
dart: ">=3.9.0 <4.0.0"
flutter: ">=3.35.0"
dart: ">=3.12.0 <4.0.0"
flutter: ">=3.44.0"

View File

@@ -2,7 +2,7 @@ name: showen_v2_flutter
description: ShowenV2 cross-platform mobile controller.
publish_to: 'none'
version: 0.1.0+1
version: 0.4.0+4
environment:
sdk: '>=3.3.0 <4.0.0'
@@ -17,6 +17,9 @@ dependencies:
provider: ^6.1.2
web_socket_channel: ^3.0.1
shared_preferences: ^2.3.0
record: ^6.0.0
audioplayers: ^6.1.0
path_provider: ^2.1.5
dev_dependencies:
flutter_test:

View File

@@ -0,0 +1,90 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:showen_v2_flutter/models/ai_model.dart';
import 'package:showen_v2_flutter/models/character_pack.dart';
import 'package:showen_v2_flutter/models/chat_models.dart';
void main() {
group('CharacterPack / AvailableConfigs', () {
test('parses M2.1 config available payload', () {
final snapshot = AvailableConfigs.fromJson({
'active': 'dog_state_machine.json',
'configs': [
{
'filename': 'dog_state_machine.json',
'character': {
'name': '小狗',
'character_type': 'pet',
'render_type': 'video',
},
},
{
'filename': 'live2d_demo.json',
'character': {
'name': 'Live2D',
'character_type': 'live2d',
'render_type': 'live2d',
'live2d_model': 'Hiyori',
},
},
],
});
expect(snapshot.active, 'dog_state_machine.json');
expect(snapshot.configs, hasLength(2));
expect(snapshot.configs.first.name, '小狗');
expect(snapshot.configs.first.typeLabel, '宠物');
expect(snapshot.configs.last.isLive2d, isTrue);
});
});
group('ChatTurnResponse', () {
test('parses success and error responses', () {
final ok = ChatTurnResponse.fromJson({
'session_id': 'app_1',
'transcription': '你好',
'reply_text': '汪!',
'reply_audio_path': '/tmp/tts.wav',
'error': null,
});
expect(ok.isOk, isTrue);
expect(ok.replyText, '汪!');
expect(ok.replyAudioPath, '/tmp/tts.wav');
final err = ChatTurnResponse.fromJson({
'session_id': '',
'reply_text': '',
'error': 'AI 插件未就绪',
});
expect(err.isOk, isFalse);
expect(err.error, 'AI 插件未就绪');
});
});
group('ModelsSnapshot', () {
test('parses model list and active map', () {
final snapshot = ModelsSnapshot.fromJson({
'models': [
{
'id': 'qwen2.5-0.5b-q4_k_m',
'kind': 'llm',
'name': 'Qwen2.5 0.5B',
'version': '1',
'size': 400000000,
'memory_required': 1000000000,
'recommended_tier': '4G',
'downloaded': true,
'download_progress': 100,
},
],
'active': {'llm': 'qwen2.5-0.5b-q4_k_m', 'asr': 'whisper-tiny'},
'used_space': 500000000,
'quota': 4000000000,
});
expect(snapshot.models, hasLength(1));
expect(snapshot.isActive(snapshot.models.first), isTrue);
expect(snapshot.usageRatio, closeTo(0.125, 0.001));
expect(snapshot.models.first.kindLabel, 'LLM');
});
});
}

204
docs/PLAN_M2.1_NEXT.md Normal file
View File

@@ -0,0 +1,204 @@
# M2.1 下一阶段实施计划
> 创建2026-07-09
> 背景功能不完善AI 与 Live2D/视频**尚未完整联动**App/Web 为半成品。
> 对照:`docs/STATUS.md`(现状)· `docs/M2.1_PRD.md`(产品定义)
---
## 0. 目标(本阶段完成的定义)
**可演示的闭环P0 验收)**
1. 设备开机 → App/Web 可连
2. 切换角色(至少狗/猫/Live2D 配置)→ 画面与人设同步
3. 文字对话 → 客户端播 TTS → **画面有“在说话”反馈**
4. App 按住说话 → 同上
5. 模型列表可见;切换 LLM 后下一回合生效(失败有明确提示)
**不做本阶段(明确边界)**
- 设备本机麦克风/扬声器直连(硬件未接)
- 精细口型同步 / 流式打断
- 数字人/歌姬全套新视频素材制作
- 云端 LLM 实装
---
## 1. 现状结论(计划前提)
| 层 | 现状 |
|----|------|
| AI 管线 | 本地 ASR/LLM/TTS 可跑HTTP `/api/chat/*` 可用 |
| Live2D | 有字幕 `Subtitle``live2d_talking` 标志在 HTTP 内,**未驱动嘴部动画** |
| 视频状态机 | 有 `talk_state` 配置字段;**对话不会 ChangeScene** |
| App | 角色/对话/模型页已写;真机旅程未系统验 |
| Web | chat 控制台有入口HTTP 录音受限 |
| 仓库 | 2026-07-09 已提交整理与骨架改动(见 git log |
---
## 2. 工作包拆分
### WP-A · AI ↔ 画面联动(核心缺口)
**A1. Live2D 说话态接通(优先)**
| 步骤 | 内容 | 验收 |
|------|------|------|
| A1.1 | 定义共享通道:消息 `Message::Live2dTalking { on: bool }` **或** 注入 Arc 状态到 Live2D 插件 | 二选一,文档写清 |
| A1.2 | HTTP chat 成功路径:开始 `talking=true`,结束按**音频时长或字数**回 `false` | 与 TTS 结束大致对齐(误差 ≤1.5s 可接受) |
| A1.3 | Live2D 渲染循环读 talking → `animation.set_talking` | 说话时嘴部开合,结束后回 idle |
| A1.4 | 字幕路径保持:`Subtitle` 继续走 | 字+嘴同时有反馈 |
**A2. 视频角色 talk 状态(次优先)**
| 步骤 | 内容 | 验收 |
|------|------|------|
| A2.1 | chat 开始:向 video 发 `ChangeScene(config.character.talk_state)` 或等价 trigger | 有 talk 片段则切过去 |
| A2.2 | chat 结束:回 `idle`(或配置的默认态) | 不卡在 talk |
| A2.3 | 无 talk 状态时优雅降级(日志 + 保持当前态,不 500 | 猫狗旧包不炸 |
**A3. 时长与互斥**
| 步骤 | 内容 |
|------|------|
| A3.1 | 优先用回复音频时长驱动 talking 窗口;无音频则字数估算 |
| A3.2 | 对话进行中拒绝二次对话 / 切角色(或排队策略写死一种) |
**负责建议**:内核/HTTP张明远 or 王浩然)+ Live2D/视频(李思琪)
**预估**A1 12 天 · A2 1 天 · A3 0.5 天
---
### WP-B · 设备对齐与回归
| 步骤 | 内容 | 验收 |
|------|------|------|
| B1 | 本机已提交代码同步到设备 `192.168.31.105` | git 一致或 rsync 可追溯 |
| B2 | `cargo check --workspace --all-targets` 零 warning | 贴输出 |
| B3 | `cargo test --workspace` 全绿 | 贴输出 |
| B4 | `systemctl --user restart showen_v2` + BLE ActiveInstances=1 | 服务稳定 |
| B5 | `POST /api/chat/text` 冒烟 | 有 reply_textLive2D 有嘴/字幕 |
---
### WP-C · App 真机验收与修补
| 步骤 | 内容 | 验收 |
|------|------|------|
| C1 | 安装 debug APK配置 `192.168.31.105:5000` | 能连 |
| C2 | 角色页切换 3 套配置 | 画面+人设变 |
| C3 | 文字对话 + 自动播 TTS + 重播 | 有声有字 |
| C4 | 按住说话一轮 | 转写+回复(允许慢) |
| C5 | 模型页列表/切换 | 失败有文案 |
| C6 | 按失败项修 Flutter超时、503、权限 | 不再静默失败 |
**依赖**WP-A 至少完成 A1 后再验“说话联动”。
---
### WP-D · Web 验收与修补
| 步骤 | 内容 |
|------|------|
| D1 | 角色/文字对话/模型三页真机点通 |
| D2 | HTTP 下录音禁用时的固定文案(与 PRD 一致) |
| D3 | 切角色清空会话上下文 |
---
### WP-E · 产品化收尾P1AD 之后)
| 步骤 | 内容 |
|------|------|
| E1 | App 对话历史本地持久化 + 一键清空 |
| E2 | 模型下载进度轮询/失败重试 UX |
| E3 | release APK 签名与安装说明 |
| E4 | M1.2 tag `m1.2`(老板) |
| E5 | 素材:数字人/歌姬 talk/idle外部依赖可并行 |
---
## 3. 推荐执行顺序
```text
Day 0 本计划评审(老板/CEO
Day 1 WP-A1 Live2D talking 接通 + B 设备同步回归
Day 2 WP-A2 视频 talk 降级策略 + A3 互斥
Day 3 WP-C App 真机四旅程 + 修洞
Day 4 WP-D Web 点通 + 文档 STATUS 更新
Day 5+ WP-E 按优先级穿插
```
---
## 4. 技术要点备忘(实现时)
### 4.1 Live2D talking 推荐实现
```
HTTP chat 开始
→ state.set_live2d_talking(true)
→ tx.send(Message::Live2dTalking { on: true }) // 或等价
→ tx.send(Message::Subtitle { text })
Live2DPlugin::handle_message
→ renderer.set_talking(on)
HTTP chat 结束spawn 延迟)
→ set_live2d_talking(false) + Live2dTalking { on: false }
```
避免 Live2D 去轮询 HTTP 内部 AtomicBool跨插件耦合差
### 4.2 视频 talk
```
render_type == video 时:
chat 开始 → PlayerCommand::ChangeScene(talk_state)
chat 结束 → ChangeScene("idle") // 或配置 default_idle
无该 scene 时: 忽略并 log不返回 5xx
```
### 4.3 验证命令(设备)
```bash
ssh -p 2222 showen@192.168.31.105
systemctl --user status showen_v2
curl -sS -m 60 -X POST http://127.0.0.1:5000/api/chat/text \
-H 'Content-Type: application/json' \
-d '{"text":"你好","session_id":"plan_test"}'
# 同时观察 Live2D 嘴部与字幕
```
---
## 5. 风险
| 风险 | 应对 |
|------|------|
| 对话延迟 515s 体感差 | 先保证正确联动;优化单独立项(锁核/更小模型) |
| 宠物包无 talk 片段 | 强制降级,不阻塞 Live2D 路径 |
| 设备与 git 漂移 | B1 强制对齐后再验 |
| 录音权限/机型差异 | App 明确权限引导;失败可走文字 |
---
## 6. 完成检查清单(勾选)
- [ ] Live2D对话时嘴部动画 + 字幕
- [ ] 视频:有 talk 则切 talk无则降级
- [ ] App角色 / 文字 / 语音 / 模型 四旅程通过
- [ ] Web角色 / 文字对话 / 模型通过
- [ ] 设备 `cargo test --workspace` 全绿
- [ ] `docs/STATUS.md` 矩阵更新为与实机一致
- [ ] 可选release APK
---
## 7. 变更记录
| 日期 | 说明 |
|------|------|
| 2026-07-09 | 初稿:联动缺口 + 分 WP 计划;对应 git 提交含骨架与整理 |

170
docs/STATUS.md Normal file
View File

@@ -0,0 +1,170 @@
# ShowenV2 项目现状板
> **更新2026-07-09**
> 用途:一页看清「能干什么 / 不能干什么 / 下一步做什么」。
> 操作派发仍以 `CLAUDE.md` 为 CEO 手册;本文件是**功能成熟度**权威清单。
---
## 1. 里程碑总览
| 里程碑 | 状态 | 说明 |
|--------|------|------|
| **M1.1** | ✅ 完成 | 插件微内核、视频/设备/HTTP/BLE/WiFi 骨架 |
| **M1.2** | ✅ 验收通过 | 目标机 147/147 测试全绿;仅差老板打 tag `m1.2` |
| **M2.1** | 🟡 **进行中(半成品)** | 语音数字生命 V1后端/网页/App 已有骨架,**体验与验收未闭环** |
---
## 2. 仓库结构(整理后)
```
ShowenV2/
├── CLAUDE.md # CEO 操作手册(派发/铁律/当前状态)
├── README.md # 架构与快速开始
├── PROGRESS.md # 里程碑摘要
├── docs/
│ ├── STATUS.md # ← 本文件(功能成熟度)
│ ├── M2.1_PRD.md # M2.1 产品需求
│ └── ...
├── src/
│ ├── core/ # 微内核
│ └── plugins/ # video / http / ble / wifi / device / screen / live2d / ai
├── configs/ # 角色内容包 JSONdog/cat/live2d_anime
├── clients/flutter/ # 安卓控制 App
├── scripts/device/ # 设备运维脚本(从根目录迁入)
├── tests/ # M1.2 集成测试
└── souls/ # 团队档案
```
**根目录禁止再堆**:截图 `*.xwd`、临时模型 `.haru.*`、一次性 `.*.sh`
---
## 3. 能力成熟度矩阵
图例:✅ 可用 · 🟡 半成品 · ❌ 缺失 · 🚫 本期不做
### 3.1 设备端Linux ARM64 · 192.168.31.105
| 能力 | 状态 | 备注 |
|------|------|------|
| 开机自启 `showen_v2.service` | ✅ | `Restart=always`,用户级 + Linger |
| 视频状态机 / Live2D 渲染 | ✅ | 设备画面主路径 |
| HTTP :5000 Web + API | ✅ | |
| BLE 广播 + GATT 配网 | ✅ | 已修 LocalName/Includes 冲突;广播实例=1 |
| WiFi (nmcli) | ✅ | |
| AI 对话管线 ASR→LLM→TTS | 🟡 | 设备实测文字对话可通;延迟数秒~十余秒 |
| 模型管理 API | 🟡 | 列表/下载/切换/删除有;进度/配额 UX 待验 |
| 角色配置切换 API | ✅ | `/api/config/available` + `switch` 带 character 元信息 |
| 单实例锁 | ❌ | 防御性双保险未做 |
| 设备本机麦克风/扬声器对话 | 🚫 | 硬件未接V1 声音走客户端 |
### 3.2 网页控制端(`chat.html` + `chat.js`
| 能力 | 状态 | 备注 |
|------|------|------|
| 播放/触发/配置/视频/文件/WiFi | ✅ | M1 存量 |
| 角色切换 UI | 🟡 | 有入口,端到端验收不足 |
| 文字对话 + 浏览器播 TTS | 🟡 | API 通;人设/状态联动待系统验 |
| 按住说话 | 🟡 | HTTP 下 getUserMedia 受限PRD 允许降级 |
| 模型管理 UI | 🟡 | 有入口,下载进度/错误态未系统验 |
| 与 App 状态互相同步 | 🟡 | 依赖 WS/轮询,未做验收用例 |
### 3.3 安卓 AppFlutter · v0.4 debug APK
| 能力 | 状态 | 备注 |
|------|------|------|
| 设备连接 / 播放 / 网络 / BLE 配网 / 设置 | ✅ | M1 能力 |
| 角色页 | 🟡 | 代码已加,**真机体验未验** |
| 对话页(文字+按住说话+播回复) | 🟡 | 代码已加,**真机体验未验** |
| 模型管理页 | 🟡 | 代码已加,**真机体验未验** |
| debug APK | ✅ | `clients/flutter/build/app/outputs/flutter-apk/app-debug.apk` (~146MB) |
| release 签名包 | ❌ | 未打 |
| 对话历史本地持久化 / 隐私文案 | ❌ | PRD 有要求 |
| 角色卡片封面图 | ❌ | API 未提供 cover |
### 3.4 内容与素材
| 项 | 状态 |
|----|------|
| dog / cat 视频包 | ✅ 可用(路径仍偏仓库外历史形态) |
| live2d_anime 包 | 🟡 有配置,素材依赖设备本地 `configs/live2d/` |
| 数字人/歌姬完整 4 状态视频包 | ❌ 素材未齐PRD 最大外部依赖) |
| 角色 `talk` 状态全覆盖 | 🟡 宠物包可降级 |
---
## 4. 未提交改动本机工作区2026-07-09
以下在仓库 **working tree**,尚未 commit按老板规则不代提交
| 区域 | 内容 |
|------|------|
| `src/plugins/ble/` | BLE 广播 LocalName 修复 + 串行注册 |
| `src/plugins/http/` | chat/audio 说话态/sessionchat.js session 头 |
| `clients/flutter/` | M2.1 三角色/对话/模型页 + Gradle/依赖升级 |
| 根目录清理 | 移除 xwd/haru 垃圾;脚本迁 `scripts/device/` |
设备端:上述 BLE/HTTP 二进制可能已部署,但 **git 与设备代码可能不完全一致**,合并前需同步。
---
## 5. 不完善点(按优先级)
### P0 — 不做则无法说「M2.1 可用」
1. **真机验收闭环**App 装机 → 角色切换 → 文字/语音对话 → 模型列表/切换
2. **设备 ↔ 本机代码对齐**commit + 设备 `git pull`/同步 + `cargo test`
3. **对话失败可理解**AI 未就绪 / 超时 / 模型缺失时App/Web 明确提示
### P1 — 体验完整度
4. App 对话历史本地存储 + 清空
5. 切换角色时清空会话上下文App 已调 clearWeb 需确认)
6. 模型下载进度实时刷新与失败重试
7. Web HTTP 录音限制的产品文案统一
8. release APK 与安装渠道
### P2 — 技术债 / 收尾
9. M1.2 tag `m1.2`
10. 单实例保护
11. `sanitize_filename` 测试转正
12. CursorVisibility 等 Phase2 消息清理
13. 数字人/歌姬素材管线
---
## 6. 建议下一轮工作顺序
```text
1) 老板/本机:提交当前工作区改动(或拆成 ble / http / flutter 三提交)
2) 设备同步 + cargo test --workspace
3) App 真机:配网 → 角色 → 对话 → 模型 四条用户旅程
4) 按失败项修 P0
5) 再谈素材与 release
```
---
## 7. 环境速查
| 环境 | 要点 |
|------|------|
| 设备 SSH | `ssh -p 2222 showen@192.168.31.105`(密码 showen |
| 设备 Web | `http://192.168.31.105:5000/` |
| 本机 Flutter | `source ~/.showen_dev_env` · Flutter 3.44.5 · JDK17 · Android SDK36 |
| 本机 APK | `clients/flutter/build/app/outputs/flutter-apk/app-debug.apk` |
---
## 8. 文档导航
| 需求 | 文件 |
|------|------|
| 派任务 / 铁律 | `CLAUDE.md` |
| 功能成熟度(本页) | `docs/STATUS.md` |
| M2.1 产品定义 | `docs/M2.1_PRD.md` |
| 里程碑摘要 | `PROGRESS.md` |
| 架构入门 | `README.md` |

15
scripts/device/README.md Normal file
View File

@@ -0,0 +1,15 @@
# 设备运维脚本
从仓库根目录迁入的临时/运维 shell供目标机调试用**非产品运行时依赖**。
| 脚本 | 用途(以内容为准) |
|------|-------------------|
| `ssh_check.sh` | SSH 连通检查 |
| `build.sh` / `run.sh` / `start.sh` | 设备侧编译/启动辅助 |
| 其余 | 历史探测脚本,用前请先读内容 |
主服务请用:
```bash
systemctl --user status showen_v2.service
```

View File

@@ -1,6 +1,7 @@
#!/bin/bash
cd /home/showen/Showen/ShowenV2
export PATH=/home/showen/.rustup/toolchains/stable-aarch64-unknown-linux-gnu/bin:$PATH
cargo check 2>&1 | grep -E "^error" | head -60
echo "---SUMMARY---"
cargo check 2>&1 | tail -3
pkill -9 -f showen_v2 2>/dev/null
sleep 1
cargo build --release 2>&1 | tail -3
echo "BUILD_EXIT=$?"

11
scripts/device/run.sh Normal file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
killall showen_v2 2>/dev/null
sleep 1
cd /home/showen/Showen/ShowenV2
DISPLAY=:0 RUST_LOG=info nohup ./target/release/showen_v2 --config configs/live2d_anime.json > /tmp/sl2.log 2>&1 &
echo "PID=$!"
sleep 8
echo "=== LOG ==="
cat /tmp/sl2.log | tail -50
echo "=== PS ==="
ps aux | grep showen_v2 | grep -v grep

12
scripts/device/start.sh Normal file
View File

@@ -0,0 +1,12 @@
#!/bin/bash
# 杀旧进程
pkill -9 -f showen_v2 2>/dev/null
sleep 1
cd /home/showen/Showen/ShowenV2
export DISPLAY=:0
export RUST_LOG=info
# 后台启动
nohup ./target/release/showen_v2 --config configs/live2d_anime.json > /tmp/sl2.log 2>&1 &
echo "PID=$!"

View File

@@ -69,6 +69,10 @@ pub enum Message {
/// AI 模型状态变更广播(加载/切换/下载进度)
AiModelEvent(AiModelEvent),
// ── 设备端字幕 (M2.1) ──
/// 设备端 Live2D 字幕显示AI 回复文字 OverlayHTTP 插件发出live2d 插件消费
Subtitle { text: String },
// ── 扩展(未来插件用) ──
Custom {
kind: String,

View File

@@ -250,6 +250,11 @@ pub fn run_ble_service(
shared: shared.clone(),
},
);
// BlueZ LEAdvertisement1 约束5.55+
// Includes 里若含 "local-name",就禁止再设 LocalName 属性,否则
// parse_local_name 报 "Local name already included" →
// "Failed to parse advertisement"。
// 我们需要自定义设备名,因此只用 LocalNameIncludes 仅保留 tx-power。
cr.insert(
ADV_PATH,
&[advertisement_iface],
@@ -257,7 +262,7 @@ pub fn run_ble_service(
advertisement_type: "peripheral".to_string(),
service_uuids: vec![SERVICE_UUID.to_string()],
local_name: device_name.clone(),
includes: vec!["tx-power".to_string(), "local-name".to_string()],
includes: vec!["tx-power".to_string()],
},
);
@@ -315,20 +320,37 @@ pub fn run_ble_service(
// 先尝试清理上一次进程残留的注册(防止崩溃后 BlueZ 状态残留)
let _ = unregister_ble_objects(&conn, &adapter_path);
// 给 BlueZ 一点时间释放残留 GATT/Adv 实例
for _ in 0..5 {
if stop.load(Ordering::SeqCst) {
return Ok(());
}
conn.process(Duration::from_millis(50)).ok();
}
// 串行注册:先 GATT再 Advertisement。并行时若 Adv 成功、GATT 失败,
// 残留 ActiveInstances 会导致后续 "Failed to create entry in database"。
let gatt_serial = send_register_gatt_app(&conn, &adapter_path)?;
let ad_serial = send_register_advertisement(&conn, &adapter_path)?;
if let Ok(mut replies) = registration_replies.lock() {
replies.gatt_serial = Some(gatt_serial);
}
eprintln!("[BLE] RegisterApplication sent, waiting...");
if let Err(error) =
wait_for_one_registration_reply(&conn, &registration_replies, "gatt", Duration::from_secs(10))
{
let _ = unregister_ble_objects(&conn, &adapter_path);
return Err(error);
}
let ad_serial = send_register_advertisement(&conn, &adapter_path)?;
if let Ok(mut replies) = registration_replies.lock() {
replies.advertisement_serial = Some(ad_serial);
}
eprintln!("[BLE] registration requests sent, waiting for BlueZ replies...");
if let Err(error) = wait_for_registration_replies(
eprintln!("[BLE] RegisterAdvertisement sent, waiting...");
if let Err(error) = wait_for_one_registration_reply(
&conn,
&registration_replies,
gatt_serial,
ad_serial,
"advertisement",
Duration::from_secs(10),
) {
let _ = unregister_ble_objects(&conn, &adapter_path);
@@ -575,36 +597,34 @@ fn replies_for_serial(
}
}
fn wait_for_registration_replies(
fn wait_for_one_registration_reply(
conn: &Connection,
replies: &Arc<Mutex<RegistrationReplies>>,
gatt_serial: u32,
advertisement_serial: u32,
which: &str,
timeout: Duration,
) -> Result<()> {
let deadline = Instant::now() + timeout;
let op = match which {
"gatt" => "RegisterApplication",
"advertisement" => "RegisterAdvertisement",
other => other,
};
loop {
if let Ok(mut replies) = replies.lock() {
match reply_status(&mut replies.gatt, "RegisterApplication")? {
Some(()) => {}
None => {}
}
match reply_status(&mut replies.advertisement, "RegisterAdvertisement")? {
Some(()) => {}
None => {}
}
if replies.gatt.is_some() && replies.advertisement.is_some() {
let slot = match which {
"gatt" => &mut replies.gatt,
"advertisement" => &mut replies.advertisement,
_ => return Err(anyhow!("unknown BLE registration target: {which}")),
};
if let Some(()) = reply_status(slot, op)? {
return Ok(());
}
}
let now = Instant::now();
if now >= deadline {
return Err(anyhow!(
"timed out waiting for BLE registration reply (gatt_serial={gatt_serial}, advertisement_serial={advertisement_serial})"
));
return Err(anyhow!("timed out waiting for BLE {op} reply"));
}
conn.process(Duration::from_millis(100).min(deadline.saturating_duration_since(now)))

View File

@@ -1,7 +1,8 @@
//! BlePlugin — BLE 配网服务
//!
//! 通过 D-Bus 与 BlueZ 交互,注册 GATT 服务和 LE Advertisement。
//! LocalName 双连接修复。
//! 注意Advertisement 不可同时 Includes["local-name"] + LocalName 属性
//! BlueZ 会报 Failed to parse advertisement
mod gatt;
@@ -95,11 +96,20 @@ impl Plugin for BlePlugin {
if stop.load(Ordering::SeqCst) {
return Ok(());
}
// BlueZ 残留状态database / advertisement需要更长冷却时间
let err_text = format!("{error:#}");
let wait_secs = if err_text.contains("database")
|| err_text.contains("parse advertisement")
|| err_text.contains("Already Exists")
{
8
} else {
3
};
eprintln!(
"[BlePlugin] worker error: {error:#}, restarting in 3 seconds..."
"[BlePlugin] worker error: {err_text}, restarting in {wait_secs} seconds..."
);
// 等待 3 秒,但如果 stop 被设置则立即退出
for _ in 0..30 {
for _ in 0..(wait_secs * 10) {
if stop.load(Ordering::SeqCst) {
return Ok(());
}

View File

@@ -65,7 +65,7 @@ async function switchModel(id,kind){try{var res=await fetch('/api/models/switch'
async function deleteModel(id){if(!confirm('删除 '+id+'?'))return;try{var res=await fetch('/api/models/delete',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model_id:id})});var data=await res.json();toast(data.message||'已删除');loadModels()}catch(e){toast('失败',true)}}
initMic();
async function initMic(){var micBtn=$('micBtn');var tip=$('recTip');if(!navigator.mediaDevices||!navigator.mediaDevices.getUserMedia){tip.innerHTML='当前浏览器不支持录音。请使用文字对话。';return}if(!window.isSecureContext){micBtn.disabled=true;tip.innerHTML='⚠️ 录音需要安全上下文HTTPS 或 localhost。当前为 HTTP 访问,录音已禁用。<br>启用方法(桌面 Chrome/Edge右键浏览器图标 → 属性 → 在目标末尾加 <code>--unsafely-treat-insecure-origin-as-secure=http://192.168.31.105:5000</code> → 重启浏览器并访问本页。<br>或直接用文字对话,回复语音仍可正常播放。';return}var mediaRecorder=null,chunks=[],recStream=null,recTimer=null;var startRec=async function(){if(chatBusy)return;try{recStream=await navigator.mediaDevices.getUserMedia({audio:{channelCount:1,sampleRate:16000,echoCancellation:true,noiseSuppression:true}});chunks=[];mediaRecorder=new MediaRecorder(recStream);mediaRecorder.ondataavailable=function(e){if(e.data.size>0)chunks.push(e.data)};mediaRecorder.onstop=function(){if(recTimer){clearTimeout(recTimer);recTimer=null}recStream.getTracks().forEach(function(t){t.stop()});recStream=null;var blob=new Blob(chunks,{type:mediaRecorder.mimeType||'audio/webm'});if(blob.size<1000){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('chatStatus').textContent='录音过短,已取消';$('chatStatus').className='status-bar error';return}sendAudioBlob(blob,mediaRecorder.mimeType||'audio/webm')};mediaRecorder.start();micBtn.classList.add('recording');micBtn.textContent='⏹';$('chatStatus').textContent='录音中...(松开发送)';$('chatStatus').className='status-bar busy';recTimer=setTimeout(function(){if(mediaRecorder&&mediaRecorder.state==='recording'){stopRec();$('chatStatus').textContent='录音达到 30s 上限,自动发送'}},30000)}catch(e){micBtn.classList.remove('recording');micBtn.textContent='🎙';$('chatStatus').textContent='麦克风启动失败: '+e.message;$('chatStatus').className='status-bar error'}};var stopRec=function(){if(mediaRecorder&&mediaRecorder.state==='recording'){mediaRecorder.stop();micBtn.classList.remove('recording');micBtn.textContent='🎙'}};micBtn.disabled=false;micBtn.addEventListener('mousedown',startRec);micBtn.addEventListener('touchstart',function(e){e.preventDefault();startRec()});micBtn.addEventListener('mouseup',stopRec);micBtn.addEventListener('mouseleave',stopRec);micBtn.addEventListener('touchend',function(e){e.preventDefault();stopRec()})}
async function sendAudioBlob(blob,mimeType){if(chatBusy)return;chatBusy=true;$('sendBtn').disabled=true;var micBtn=$('micBtn');micBtn.disabled=true;$('chatStatus').textContent='转码中...';$('chatStatus').className='status-bar busy';var userMsg=addMsg('user','🎤 [语音消息]');var playUrl=URL.createObjectURL(blob);var up=document.createElement('div');up.className='audio-inline';up.innerHTML='<span style="font-size:11px;color:var(--muted)">你的录音</span><audio controls src="'+playUrl+'"></audio>';userMsg.appendChild(up);if(currentChar.render_type!=='live2d'&&currentChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);var botMsg=addMsg('bot','思考中...');try{var wavBlob=await blobToWav(blob);$('chatStatus').textContent='识别中...';var res=await fetch('/api/chat/audio',{method:'POST',headers:{'Content-Type':'audio/wav','X-Audio-Format':'wav'},body:wavBlob});var data=await res.json();handleChatResponse(data,botMsg)}catch(e){showChatError(botMsg,'请求失败: '+e.message)}chatBusy=false;$('sendBtn').disabled=false;micBtn.disabled=false}
async function sendAudioBlob(blob,mimeType){if(chatBusy)return;chatBusy=true;$('sendBtn').disabled=true;var micBtn=$('micBtn');micBtn.disabled=true;$('chatStatus').textContent='转码中...';$('chatStatus').className='status-bar busy';var userMsg=addMsg('user','🎤 [语音消息]');var playUrl=URL.createObjectURL(blob);var up=document.createElement('div');up.className='audio-inline';up.innerHTML='<span style="font-size:11px;color:var(--muted)">你的录音</span><audio controls src="'+playUrl+'"></audio>';userMsg.appendChild(up);if(currentChar.render_type!=='live2d'&&currentChar.talk_state)triggerTalkState(true);if(currentChar.render_type==='live2d')showLive2dTalking(true);var botMsg=addMsg('bot','思考中...');try{var wavBlob=await blobToWav(blob);$('chatStatus').textContent='识别中...';var res=await fetch('/api/chat/audio',{method:'POST',headers:{'Content-Type':'audio/wav','X-Audio-Format':'wav','X-Session-Id':sessionId},body:wavBlob});var data=await res.json();handleChatResponse(data,botMsg)}catch(e){showChatError(botMsg,'请求失败: '+e.message)}chatBusy=false;$('sendBtn').disabled=false;micBtn.disabled=false}
/* WAV 转码:浏览器录音(webm/opus) → 16k mono PCM16 wavwhisper-cli 直接可处理 */
async function blobToWav(blob){var arrayBuf=await blob.arrayBuffer();var audioCtx=new(window.AudioContext||window.webkitAudioContext)({sampleRate:16000});var decoded=await audioCtx.decodeAudioData(arrayBuf);var pcm=decodeToMonoPCM16(decoded,16000);audioCtx.close();return encodeWavBlob(pcm,16000)}
function decodeToMonoPCM16(audioBuffer,targetRate){var numCh=audioBuffer.numberOfChannels;var len=audioBuffer.length;var srcRate=audioBuffer.sampleRate;var outLen=Math.round(len*targetRate/srcRate);var out=new Int16Array(outLen);var tmp=new Float32Array(outLen);for(var ch=0;ch<numCh;ch++){var data=audioBuffer.getChannelData(ch);for(var i=0;i<outLen;i++){var srcIdx=i*srcRate/targetRate;var i0=Math.floor(srcIdx);var i1=Math.min(i0+1,len-1);var frac=srcIdx-i0;tmp[i]+=data[i0]*(1-frac)+data[i1]*frac}}for(var i=0;i<outLen;i++){var s=tmp[i]/numCh;s=Math.max(-1,Math.min(1,s));out[i]=s<0?s*0x8000:s*0x7FFF}return out}

View File

@@ -5,11 +5,12 @@
mod routes;
use crate::core::config::AppConfig;
use crate::core::message::{Envelope, Message};
use crate::core::message::{Destination, Envelope, Message};
use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo};
use anyhow::{Context, Result};
use serde::Serialize;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::sync::{Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use tokio::sync::broadcast;
@@ -54,10 +55,12 @@ pub(crate) struct HttpState {
ai_models: Mutex<Option<std::sync::Arc<std::sync::Mutex<crate::plugins::ai::model_manager::ModelManager>>>>,
/// Live2D 说话状态(设备端 Firefox 轮询,驱动嘴部动效)
live2d_talking: AtomicBool,
/// 向其他插件(如 live2d转发消息的出口init 时由 PluginContext.tx 注入)
outbox: mpsc::Sender<Envelope>,
}
impl HttpState {
fn new(config: Arc<AppConfig>) -> Self {
fn new(config: Arc<AppConfig>, tx: mpsc::Sender<Envelope>) -> Self {
let (ws_events, _) = broadcast::channel(32);
let player_status = crate::core::message::PlayerStatusData {
running: false,
@@ -84,6 +87,18 @@ impl HttpState {
ai_pipeline: Mutex::new(None),
ai_models: Mutex::new(None),
live2d_talking: AtomicBool::new(false),
outbox: tx,
}
}
/// 向指定插件发送消息(用于把 AI 回复文字推给 live2d 做设备端字幕)。
pub(crate) fn send_message(&self, to: Destination, msg: Message) {
if let Err(e) = self.outbox.send(Envelope {
from: "http".to_string(),
to,
message: msg,
}) {
eprintln!("[HttpPlugin] 发送消息失败: {e}");
}
}
@@ -312,7 +327,10 @@ impl Plugin for HttpPlugin {
}
fn init(&mut self, ctx: PluginContext) -> Result<()> {
let state = Arc::new(HttpState::new(Arc::clone(&ctx.config)));
let state = Arc::new(HttpState::new(
Arc::clone(&ctx.config),
ctx.tx.clone(),
));
// 注册 AI 句柄(如果 main.rs 已设置)
if let Some(pipeline) = self.ai_pipeline.take() {
state.register_ai_pipeline(pipeline);

View File

@@ -2188,6 +2188,16 @@ fn chat_text_route(
reset_state.set_live2d_talking(false);
});
// 设备端字幕:把回复文字推给 live2d 插件做屏幕 Overlay
if resp.error.is_none() && !resp.reply_text.is_empty() {
state.send_message(
Destination::Plugin("live2d".to_string()),
Message::Subtitle {
text: resp.reply_text.clone(),
},
);
}
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {
@@ -2201,6 +2211,10 @@ fn chat_text_route(
/// 注意:上传的音频会被传给 whisper-cli它通过文件扩展名判断格式。
/// 浏览器 MediaRecorder 默认产 webm/opus移动端可能产 m4a/aac。
/// 服务端不强校验格式,但建议客户端尽量转 wav/16k 单声道。
///
/// 可选头:
/// - `X-Audio-Format`: webm|ogg|m4a|mp3|wav
/// - `X-Session-Id`: 会话 ID多端一致上下文
fn chat_audio_route(
state: Arc<HttpState>,
) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone {
@@ -2209,10 +2223,12 @@ fn chat_audio_route(
.and(warp::body::content_length_limit(20 * 1024 * 1024))
.and(warp::body::bytes())
.and(warp::header::optional::<String>("X-Audio-Format"))
.and(warp::header::optional::<String>("X-Session-Id"))
.and(with_state(state))
.and_then(
|bytes: bytes::Bytes,
fmt_hint: Option<String>,
session_hint: Option<String>,
state: Arc<HttpState>| async move {
let pipeline = match state.ai_pipeline() {
Some(p) => p,
@@ -2250,7 +2266,9 @@ fn chat_audio_route(
));
}
let session_id = format!("app_{}", std::process::id());
let session_id = session_hint
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| format!("app_{}", std::process::id()));
let chat_req = ChatRequest {
session_id,
input: ChatInput::Audio {
@@ -2260,6 +2278,10 @@ fn chat_audio_route(
max_tokens: config.character.max_tokens,
};
// 与文字对话一致:驱动 Live2D 说话态
let talking_state = Arc::clone(&state);
talking_state.set_live2d_talking(true);
let audio_path_clone = audio_path.clone();
let resp = tokio::task::spawn_blocking(move || pipeline.run(&chat_req))
.await
@@ -2274,6 +2296,31 @@ fn chat_audio_route(
// 清理上传的临时音频
let _ = std::fs::remove_file(&audio_path_clone);
let reset_after = if resp.error.is_none() {
std::time::Duration::from_millis(
((resp.reply_text.len() as u64) * 200).max(2000),
)
} else {
std::time::Duration::from_millis(0)
};
let reset_state = Arc::clone(&state);
tokio::spawn(async move {
if !reset_after.is_zero() {
tokio::time::sleep(reset_after).await;
}
reset_state.set_live2d_talking(false);
});
// 设备端字幕:把回复文字推给 live2d 插件做屏幕 Overlay
if resp.error.is_none() && !resp.reply_text.is_empty() {
state.send_message(
Destination::Plugin("live2d".to_string()),
Message::Subtitle {
text: resp.reply_text.clone(),
},
);
}
if resp.error.is_some() {
Ok(json_response(StatusCode::INTERNAL_SERVER_ERROR, &resp))
} else {

View File

@@ -3,7 +3,7 @@
//! 直接对应 `Live2DCubismCore.h` 的 C API。
//! 动态加载 libLive2DCubismCore.so需 libm.so 预加载,因 Core 内部使用 powf
use libloading::{Library, Symbol};
use libloading::Library;
use std::os::raw::{c_char, c_int, c_uchar, c_uint, c_void};
/// moc 对齐要求(字节)
@@ -32,6 +32,7 @@ pub mod dynamic_flags {
/// 混合模式
pub mod blend_mode {
use std::os::raw::c_int;
pub const NORMAL: c_int = 0;
pub const ADDITIVE: c_int = 1;
pub const MULTIPLICATIVE: c_int = 2;
@@ -90,6 +91,8 @@ pub struct CubismCore {
unsafe extern "C" fn(model: *const c_void) -> *const f32,
pub csm_get_parameter_values: unsafe extern "C" fn(model: *mut c_void) -> *mut f32,
pub csm_get_part_count: unsafe extern "C" fn(model: *const c_void) -> c_int,
pub csm_get_part_ids:
unsafe extern "C" fn(model: *const c_void) -> *const *const c_char,
pub csm_get_part_opacities: unsafe extern "C" fn(model: *mut c_void) -> *mut f32,
pub csm_get_drawable_count: unsafe extern "C" fn(model: *const c_void) -> c_int,
pub csm_get_drawable_ids:
@@ -162,6 +165,7 @@ impl CubismCore {
.get(b"csmGetParameterDefaultValues\0")?,
csm_get_parameter_values: *lib.get(b"csmGetParameterValues\0")?,
csm_get_part_count: *lib.get(b"csmGetPartCount\0")?,
csm_get_part_ids: *lib.get(b"csmGetPartIds\0")?,
csm_get_part_opacities: *lib.get(b"csmGetPartOpacities\0")?,
csm_get_drawable_count: *lib.get(b"csmGetDrawableCount\0")?,
csm_get_drawable_ids: *lib.get(b"csmGetDrawableIds\0")?,

View File

@@ -21,6 +21,8 @@ pub mod egl {
pub const EGL_OPENGL_ES2_BIT: c_int = 0x0004;
pub const EGL_NONE: c_int = 0x3038;
pub const EGL_CONTEXT_CLIENT_VERSION: c_int = 0x3098;
pub const EGL_DEPTH_SIZE: c_int = 0x3025;
pub const EGL_STENCIL_SIZE: c_int = 0x3026;
pub type EGLDisplay = *mut c_void;
pub type EGLConfig = *mut c_void;
@@ -40,7 +42,7 @@ pub mod egl {
// ============ X11 ============
pub mod x11 {
use std::os::raw::{c_int, c_long, c_uint, c_void};
use std::os::raw::{c_int, c_long, c_void};
pub type Display = c_void;
pub type Window = c_long;
@@ -68,18 +70,40 @@ pub mod x11 {
pub const CWEventMask: c_long = 1 << 11;
pub const ExposureMask: c_long = 1 << 15;
pub const StructureNotifyMask: c_long = 1 << 17;
pub const SubstructureNotifyMask: c_long = 1 << 19;
pub const SubstructureRedirectMask: c_long = 1 << 20;
/// XEvent union简化为最大 192 字节的字节数组)
#[repr(C)]
pub struct XEvent {
pub data: [std::os::raw::c_char; 192],
}
/// XClientMessageEvent 布局(用于 _NET_WM_STATE 客户端消息)
#[repr(C)]
pub struct XClientMessageEvent {
pub ftype: c_long, // type (ClientMessage = 33)
pub serial: c_long, // serial
pub send_event: c_int, // send_event
pub display: *mut c_void, // display
pub window: c_long, // window
pub message_type: c_long, // message_type atom
pub format: c_int, // format (32)
pub data_l: [c_long; 5], // data.l
}
}
// ============ GLES2 ============
pub mod gl {
use std::os::raw::{c_int, c_uint};
use std::os::raw::{c_int, c_uint, c_uchar};
pub type GLenum = c_uint;
pub type GLuint = c_uint;
pub type GLint = c_int;
pub type GLsizei = c_int;
pub type GLbitfield = c_uint;
pub type GLboolean = c_uchar;
pub const GL_FLOAT: GLenum = 0x1406;
pub const GL_ARRAY_BUFFER: GLenum = 0x8892;
@@ -100,6 +124,7 @@ pub mod gl {
pub const GL_COMPILE_STATUS: GLenum = 0x8B81;
pub const GL_LINK_STATUS: GLenum = 0x8B82;
pub const GL_TRIANGLES: GLenum = 0x0004;
pub const GL_TRIANGLE_STRIP: GLenum = 0x0005;
pub const GL_BLEND: GLenum = 0x0BE2;
pub const GL_SRC_ALPHA: GLenum = 0x0302;
pub const GL_ONE_MINUS_SRC_ALPHA: GLenum = 0x0303;
@@ -110,6 +135,20 @@ pub mod gl {
pub const GL_STATIC_DRAW: GLenum = 0x88E4;
pub const GL_FALSE: GLenum = 0;
pub const GL_NO_ERROR: GLenum = 0;
// 遮罩stencil相关常量
pub const GL_DEPTH_TEST: GLenum = 0x0B71;
pub const GL_STENCIL_TEST: GLenum = 0x0B90;
pub const GL_STENCIL_BUFFER_BIT: GLbitfield = 0x0400;
pub const GL_KEEP: GLenum = 0x1E00;
pub const GL_REPLACE: GLenum = 0x1E01;
pub const GL_INCR: GLenum = 0x1E02;
pub const GL_DECR: GLenum = 0x1E03;
pub const GL_ALWAYS: GLenum = 0x0207;
pub const GL_EQUAL: GLenum = 0x0202;
pub const GL_NOTEQUAL: GLenum = 0x0205;
pub const GL_LESS: GLenum = 0x0201;
pub const GL_TRUE: GLboolean = 1;
}
pub use gl::{GLenum, GLuint, GLint, GLsizei};
@@ -140,7 +179,82 @@ void main() {
texColor.rgb = texColor.rgb * u_multiplyColor.rgb;
texColor.rgb = texColor.rgb + u_screenColor.rgb - (texColor.rgb * u_screenColor.rgb);
vec4 color = texColor * u_baseColor;
gl_FragColor = vec4(color.rgb * color.a, color.a);
gl_FragColor = vec4(color.rgb, color.a);
}
"#;
/// 背景渐变着色器(全屏四边形,从底部到底部到顶部做线性插值)
pub const GRAD_VERT_SHADER_SRC: &str = r#"#version 100
attribute vec2 a_pos;
varying float v_y;
void main() {
v_y = a_pos.y;
gl_Position = vec4(a_pos, 0.0, 1.0);
}
"#;
pub const GRAD_FRAG_SHADER_SRC: &str = r#"#version 100
precision highp float;
varying float v_y;
uniform vec4 u_top;
uniform vec4 u_bottom;
void main() {
float t = (v_y + 1.0) * 0.5;
vec3 c = mix(u_bottom.rgb, u_top.rgb, t);
gl_FragColor = vec4(c, 1.0);
}
"#;
/// 文字 overlay 着色器(像素坐标 -> 裁剪空间,采样字形纹理,按 u_color 着色)
pub const TEXT_VERT_SHADER_SRC: &str = r#"#version 100
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform mat4 u_matrix;
varying vec2 v_uv;
void main() {
gl_Position = u_matrix * vec4(a_position, 0.0, 1.0);
v_uv = a_texCoord;
}
"#;
pub const TEXT_FRAG_SHADER_SRC: &str = r#"#version 100
precision mediump float;
varying vec2 v_uv;
uniform sampler2D s_texture0;
uniform vec4 u_color;
void main() {
float a = texture2D(s_texture0, v_uv).a;
gl_FragColor = vec4(u_color.rgb, a * u_color.a);
}
"#;
/// 遮罩mask着色器仅把 mask drawable 的不透明区域写入 stencil
/// 用于裁切被遮罩的 drawable如夹在身体里的后臂、被头发遮挡的脸
pub const MASK_VERT_SHADER_SRC: &str = r#"#version 100
attribute vec4 a_position;
attribute vec2 a_texCoord;
varying vec2 v_texCoord;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * a_position;
v_texCoord = a_texCoord;
v_texCoord.y = 1.0 - v_texCoord.y;
}
"#;
pub const MASK_FRAG_SHADER_SRC: &str = r#"#version 100
precision highp float;
varying vec2 v_texCoord;
uniform sampler2D s_texture0;
uniform vec4 u_multiplyColor;
uniform vec4 u_screenColor;
void main() {
vec4 c = texture2D(s_texture0, v_texCoord);
c.rgb = c.rgb * u_multiplyColor.rgb;
c.rgb = c.rgb + u_screenColor.rgb - (c.rgb * u_screenColor.rgb);
// 仅不透明区域写入 stencil裁切形状由 alpha 决定)
if (c.a < 0.5) discard;
gl_FragColor = vec4(1.0);
}
"#;
@@ -164,6 +278,12 @@ pub struct Gl {
config_size: egl::EGLint,
num_config: *mut egl::EGLint,
) -> egl::EGLBoolean,
pub egl_get_config_attrib: unsafe extern "C" fn(
dpy: egl::EGLDisplay,
config: egl::EGLConfig,
attribute: egl::EGLint,
value: *mut egl::EGLint,
) -> egl::EGLBoolean,
pub egl_create_window_surface: unsafe extern "C" fn(
dpy: egl::EGLDisplay,
config: egl::EGLConfig,
@@ -188,6 +308,7 @@ pub struct Gl {
pub gl_clear: unsafe extern "C" fn(mask: gl::GLbitfield),
pub gl_clear_color: unsafe extern "C" fn(r: f32, g: f32, b: f32, a: f32),
pub gl_enable: unsafe extern "C" fn(cap: gl::GLenum),
pub gl_disable: unsafe extern "C" fn(cap: gl::GLenum),
pub gl_blend_func: unsafe extern "C" fn(sfactor: gl::GLenum, dfactor: gl::GLenum),
pub gl_viewport: unsafe extern "C" fn(x: gl::GLint, y: gl::GLint, w: gl::GLsizei, h: gl::GLsizei),
pub gl_create_shader: unsafe extern "C" fn(t: gl::GLenum) -> gl::GLuint,
@@ -269,8 +390,18 @@ pub struct Gl {
type_: gl::GLenum,
indices: *const c_void_raw,
),
pub gl_draw_arrays: unsafe extern "C" fn(mode: gl::GLenum, first: gl::GLint, count: gl::GLsizei),
pub gl_get_error: unsafe extern "C" fn() -> gl::GLenum,
pub gl_delete_buffers: unsafe extern "C" fn(n: gl::GLsizei, buffers: *const gl::GLuint),
pub gl_stencil_func: unsafe extern "C" fn(func: gl::GLenum, ref_: gl::GLint, mask: gl::GLuint),
pub gl_stencil_op: unsafe extern "C" fn(fail: gl::GLenum, zfail: gl::GLenum, zpass: gl::GLenum),
pub gl_stencil_mask: unsafe extern "C" fn(mask: gl::GLuint),
pub gl_color_mask: unsafe extern "C" fn(
r: gl::GLboolean,
g: gl::GLboolean,
b: gl::GLboolean,
a: gl::GLboolean,
),
pub x_open_display: unsafe extern "C" fn(name: *const c_char) -> *mut x11::Display,
pub x_create_window: unsafe extern "C" fn(
@@ -291,6 +422,33 @@ pub struct Gl {
pub x_store_name:
unsafe extern "C" fn(disp: *mut x11::Display, w: x11::Window, name: *const c_char) -> c_int,
pub x_flush: unsafe extern "C" fn(disp: *mut x11::Display) -> c_int,
pub x_default_root_window: unsafe extern "C" fn(disp: *mut x11::Display) -> x11::Window,
pub x_sync: unsafe extern "C" fn(disp: *mut x11::Display, discard: c_int) -> c_int,
pub x_raise_window: unsafe extern "C" fn(disp: *mut x11::Display, w: x11::Window) -> c_int,
pub x_display_width: unsafe extern "C" fn(disp: *mut x11::Display, screen: c_int) -> c_int,
pub x_display_height: unsafe extern "C" fn(disp: *mut x11::Display, screen: c_int) -> c_int,
pub x_intern_atom: unsafe extern "C" fn(
disp: *mut x11::Display,
atom_name: *const c_char,
only_if_exists: c_int,
) -> c_long,
pub x_change_property: unsafe extern "C" fn(
disp: *mut x11::Display,
w: x11::Window,
property: c_long,
ftype: c_long,
format: c_int,
mode: c_int,
data: *const c_void,
nelements: c_int,
) -> c_int,
pub x_send_event: unsafe extern "C" fn(
disp: *mut x11::Display,
w: x11::Window,
propagate: c_int,
event_mask: c_long,
event_send: *mut x11::XEvent,
) -> c_int,
}
impl Gl {
@@ -304,6 +462,7 @@ impl Gl {
egl_get_display: *lib_egl.get(b"eglGetDisplay\0")?,
egl_initialize: *lib_egl.get(b"eglInitialize\0")?,
egl_choose_config: *lib_egl.get(b"eglChooseConfig\0")?,
egl_get_config_attrib: *lib_egl.get(b"eglGetConfigAttrib\0")?,
egl_create_window_surface: *lib_egl.get(b"eglCreateWindowSurface\0")?,
egl_create_context: *lib_egl.get(b"eglCreateContext\0")?,
egl_make_current: *lib_egl.get(b"eglMakeCurrent\0")?,
@@ -312,6 +471,7 @@ impl Gl {
gl_clear: *lib_gl.get(b"glClear\0")?,
gl_clear_color: *lib_gl.get(b"glClearColor\0")?,
gl_enable: *lib_gl.get(b"glEnable\0")?,
gl_disable: *lib_gl.get(b"glDisable\0")?,
gl_blend_func: *lib_gl.get(b"glBlendFunc\0")?,
gl_viewport: *lib_gl.get(b"glViewport\0")?,
gl_create_shader: *lib_gl.get(b"glCreateShader\0")?,
@@ -341,14 +501,27 @@ impl Gl {
gl_enable_vertex_attrib_array: *lib_gl.get(b"glEnableVertexAttribArray\0")?,
gl_vertex_attrib_pointer: *lib_gl.get(b"glVertexAttribPointer\0")?,
gl_draw_elements: *lib_gl.get(b"glDrawElements\0")?,
gl_draw_arrays: *lib_gl.get(b"glDrawArrays\0")?,
gl_get_error: *lib_gl.get(b"glGetError\0")?,
gl_delete_buffers: *lib_gl.get(b"glDeleteBuffers\0")?,
gl_stencil_func: *lib_gl.get(b"glStencilFunc\0")?,
gl_stencil_op: *lib_gl.get(b"glStencilOp\0")?,
gl_stencil_mask: *lib_gl.get(b"glStencilMask\0")?,
gl_color_mask: *lib_gl.get(b"glColorMask\0")?,
x_open_display: *lib_x11.get(b"XOpenDisplay\0")?,
x_create_window: *lib_x11.get(b"XCreateWindow\0")?,
x_map_window: *lib_x11.get(b"XMapWindow\0")?,
x_store_name: *lib_x11.get(b"XStoreName\0")?,
x_flush: *lib_x11.get(b"XFlush\0")?,
x_default_root_window: *lib_x11.get(b"XDefaultRootWindow\0")?,
x_sync: *lib_x11.get(b"XSync\0")?,
x_raise_window: *lib_x11.get(b"XRaiseWindow\0")?,
x_display_width: *lib_x11.get(b"XDisplayWidth\0")?,
x_display_height: *lib_x11.get(b"XDisplayHeight\0")?,
x_intern_atom: *lib_x11.get(b"XInternAtom\0")?,
x_change_property: *lib_x11.get(b"XChangeProperty\0")?,
x_send_event: *lib_x11.get(b"XSendEvent\0")?,
_lib_egl: lib_egl,
_lib_gl: lib_gl,
@@ -364,6 +537,7 @@ pub struct RenderContext {
pub gl: Gl,
pub display: egl::EGLDisplay,
pub surface: egl::EGLSurface,
pub context: egl::EGLContext,
pub x_display: *mut x11::Display,
pub x_window: x11::Window,
pub program: GLuint,
@@ -374,6 +548,25 @@ pub struct RenderContext {
pub uniform_base_color: GLint,
pub uniform_multiply_color: GLint,
pub uniform_screen_color: GLint,
pub grad_program: GLuint,
pub grad_attrib_pos: GLint,
pub grad_uniform_top: GLint,
pub grad_uniform_bottom: GLint,
// 遮罩mask程序
pub mask_program: GLuint,
pub mask_attrib_position: GLint,
pub mask_attrib_tex_coord: GLint,
pub mask_uniform_matrix: GLint,
pub mask_uniform_texture: GLint,
pub mask_uniform_multiply_color: GLint,
pub mask_uniform_screen_color: GLint,
// 文字 overlay 程序
pub text_program: GLuint,
pub text_uniform_matrix: GLint,
pub text_uniform_color: GLint,
pub text_uniform_texture: GLint,
pub text_attrib_position: GLint,
pub text_attrib_tex_coord: GLint,
pub width: i32,
pub height: i32,
}
@@ -395,13 +588,10 @@ fn compile_shader(gl: &Gl, shader_type: GLenum, src: &str) -> Result<GLuint> {
let mut status = 0;
(gl.gl_get_shaderiv)(shader, gl::GL_COMPILE_STATUS, &mut status);
if status == 0 {
let mut log = vec![0i8; 1024];
(gl.gl_get_shader_info_log)(shader, 1024, std::ptr::null_mut(), log.as_mut_ptr());
let log_str = String::from_utf8_lossy(std::slice::from_raw_parts(
log.as_ptr() as *const u8,
1024,
));
return Err(anyhow!("着色器编译失败: {}", log_str));
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_shader_info_log)(shader, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
let log_str = String::from_utf8_lossy(&log);
return Err(anyhow!("shader compile failed: {}", log_str));
}
Ok(shader)
}
@@ -414,8 +604,21 @@ impl RenderContext {
// X11 窗口
let x_display = unsafe { (gl.x_open_display)(std::ptr::null()) };
if x_display.is_null() {
return Err(anyhow!("XOpenDisplay 失败"));
return Err(anyhow!("XOpenDisplay failed"));
}
let root_window = unsafe { (gl.x_default_root_window)(x_display) };
// 使用屏幕实际分辨率,确保全屏覆盖
let screen = 0; // XDefaultScreen(display),通常 0
let screen_w = unsafe { (gl.x_display_width)(x_display, screen) };
let screen_h = unsafe { (gl.x_display_height)(x_display, screen) };
// 全屏渲染:直接使用显示器实际分辨率创建窗口与 EGL surface避免
// 「帧缓冲 800x800 被放进 1920x1080 窗口、模型只出现在左下角」的问题。
// 若 X11 取不到屏幕分辨率,再退回传入的 width/height。
let win_w = if screen_w > 0 && screen_h > 0 { screen_w } else { width };
let win_h = if screen_w > 0 && screen_h > 0 { screen_h } else { height };
println!("[Live2D] 窗口尺寸: {}x{} (屏幕实际 {}x{})", win_w, win_h, screen_w, screen_h);
let mut attrs = x11::XSetWindowAttributes {
background_pixmap: 0,
background_pixel: 0,
@@ -429,34 +632,66 @@ impl RenderContext {
save_under: 0,
event_mask: x11::ExposureMask | x11::StructureNotifyMask,
do_not_propagate_mask: 0,
override_redirect: 1,
override_redirect: 0,
colormap: 0,
cursor: 0,
};
let x_window = unsafe {
(gl.x_create_window)(
x_display,
root_window,
0,
0,
0,
width as c_uint,
height as c_uint,
win_w as c_uint,
win_h as c_uint,
0,
24,
1,
std::ptr::null_mut(),
x11::CWOverrideRedirect | x11::CWEventMask,
x11::CWEventMask,
&mut attrs,
)
};
if x_window == 0 {
return Err(anyhow!("XCreateWindow 失败"));
return Err(anyhow!("XCreateWindow failed"));
}
unsafe {
let title = std::ffi::CString::new("ShowenV2 Live2D")?;
(gl.x_store_name)(x_display, x_window, title.as_ptr());
// 让 kwin 把窗口设为全屏_NET_WM_STATE_FULLSCREEN
// 通过发送 ClientMessage 事件给 root window
let net_wm_state = (gl.x_intern_atom)(x_display, b"_NET_WM_STATE\0".as_ptr() as *const c_char, 1);
let net_wm_state_fs = (gl.x_intern_atom)(x_display, b"_NET_WM_STATE_FULLSCREEN\0".as_ptr() as *const c_char, 1);
let net_wm_state_above = (gl.x_intern_atom)(x_display, b"_NET_WM_STATE_ABOVE\0".as_ptr() as *const c_char, 1);
// 先 map 窗口kwin 才会处理
(gl.x_map_window)(x_display, x_window);
(gl.x_flush)(x_display);
(gl.x_sync)(x_display, 0);
// 发送 _NET_WM_STATE 客户端消息两个状态FULLSCREEN + ABOVE
let mut event: x11::XClientMessageEvent = std::mem::zeroed();
event.ftype = 33; // ClientMessage
event.display = x_display as *mut c_void;
event.window = x_window;
event.message_type = net_wm_state;
event.format = 32;
event.data_l[0] = 1; // _NET_WM_STATE_ADD
event.data_l[1] = net_wm_state_fs;
event.data_l[2] = net_wm_state_above;
event.data_l[3] = 1; // source indication: application
let event_mask = x11::SubstructureNotifyMask | x11::SubstructureRedirectMask;
(gl.x_send_event)(
x_display,
root_window,
0,
event_mask,
&mut event as *mut x11::XClientMessageEvent as *mut x11::XEvent,
);
(gl.x_sync)(x_display, 0);
println!("[Live2D] 已请求 kwin 全屏置顶");
}
// EGL
@@ -471,7 +706,20 @@ impl RenderContext {
}
println!("[Live2D] EGL {}.{}", major, minor);
// 主配置请求 depth+stencilLive2D 遮罩需要 stencil
// 若设备不支持该组合,退回不带 depth/stencil 的配置。
let config_attrs = [
egl::EGL_RED_SIZE, 8,
egl::EGL_GREEN_SIZE, 8,
egl::EGL_BLUE_SIZE, 8,
egl::EGL_ALPHA_SIZE, 8,
egl::EGL_SURFACE_TYPE, egl::EGL_WINDOW_BIT,
egl::EGL_RENDERABLE_TYPE, egl::EGL_OPENGL_ES2_BIT,
egl::EGL_DEPTH_SIZE, 16,
egl::EGL_STENCIL_SIZE, 8,
egl::EGL_NONE,
];
let config_attrs_fallback = [
egl::EGL_RED_SIZE, 8,
egl::EGL_GREEN_SIZE, 8,
egl::EGL_BLUE_SIZE, 8,
@@ -482,18 +730,38 @@ impl RenderContext {
];
let mut config: egl::EGLConfig = std::ptr::null_mut();
let mut num_config = 0;
if unsafe {
(gl.egl_choose_config)(
display,
config_attrs.as_ptr(),
&mut config,
1,
&mut num_config,
)
} == 0 || num_config == 0
{
let mut choose = |attrs: &[c_int]| -> bool {
let mut cfg: egl::EGLConfig = std::ptr::null_mut();
let mut n = 0;
let ok = unsafe {
(gl.egl_choose_config)(display, attrs.as_ptr(), &mut cfg, 1, &mut n)
} != 0 && n > 0;
if ok {
config = cfg;
num_config = n;
}
ok
};
if !choose(&config_attrs) && !choose(&config_attrs_fallback) {
return Err(anyhow!("eglChooseConfig 失败"));
}
println!(
"[Live2D] EGL 配置选定: 含stencil={} (num_config={})",
config_attrs.len() > 9,
num_config
);
// 真实查询 chosen config 的 depth/stencil 位数,确认 stencil 缓冲真正分配。
let mut stencil_size: egl::EGLint = -1;
let mut depth_size: egl::EGLint = -1;
unsafe {
(gl.egl_get_config_attrib)(display, config, egl::EGL_STENCIL_SIZE, &mut stencil_size);
(gl.egl_get_config_attrib)(display, config, egl::EGL_DEPTH_SIZE, &mut depth_size);
}
println!(
"[Live2D] EGL 实际分配: depth={} stencil={} (stencil<=0 表示遮罩不可用)",
depth_size, stencil_size
);
let surface = unsafe {
(gl.egl_create_window_surface)(display, config, x_window as egl::NativeWindowType, std::ptr::null())
@@ -503,7 +771,7 @@ impl RenderContext {
}
let ctx_attrs = [egl::EGL_CONTEXT_CLIENT_VERSION, 2, egl::EGL_NONE];
let context = unsafe { (gl.egl_create_context)(display, config, std::ptr::null(), ctx_attrs.as_ptr()) };
let context = unsafe { (gl.egl_create_context)(display, config, std::ptr::null_mut(), ctx_attrs.as_ptr()) };
if context.is_null() {
return Err(anyhow!("eglCreateContext 失败"));
}
@@ -513,7 +781,7 @@ impl RenderContext {
}
unsafe {
(gl.gl_viewport)(0, 0, width, height);
(gl.gl_viewport)(0, 0, win_w, win_h);
(gl.gl_clear_color)(0.0, 0.0, 0.0, 1.0);
(gl.gl_enable)(gl::GL_BLEND);
(gl.gl_blend_func)(gl::GL_SRC_ALPHA, gl::GL_ONE_MINUS_SRC_ALPHA);
@@ -529,13 +797,10 @@ impl RenderContext {
let mut status = 0;
(gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status);
if status == 0 {
let mut log = vec![0i8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr());
let log_str = String::from_utf8_lossy(std::slice::from_raw_parts(
log.as_ptr() as *const u8,
1024,
));
return Err(anyhow!("着色器链接失败: {}", log_str));
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
let log_str = String::from_utf8_lossy(&log);
return Err(anyhow!("shader link failed: {}", log_str));
}
p
};
@@ -558,16 +823,131 @@ impl RenderContext {
let uniform_multiply_color = getu("u_multiplyColor")?;
let uniform_screen_color = getu("u_screenColor")?;
// 背景渐变程序
let grad_vert = compile_shader(&gl, gl::GL_VERTEX_SHADER, GRAD_VERT_SHADER_SRC)?;
let grad_frag = compile_shader(&gl, gl::GL_FRAGMENT_SHADER, GRAD_FRAG_SHADER_SRC)?;
let grad_program = unsafe {
let p = (gl.gl_create_program)();
(gl.gl_attach_shader)(p, grad_vert);
(gl.gl_attach_shader)(p, grad_frag);
(gl.gl_link_program)(p);
let mut status = 0;
(gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status);
if status == 0 {
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
return Err(anyhow!("gradient shader link failed: {}", String::from_utf8_lossy(&log)));
}
p
};
let grad_attrib_pos = {
let c = std::ffi::CString::new("a_pos")?;
unsafe { (gl.gl_get_attrib_location)(grad_program, c.as_ptr()) }
};
let grad_uniform_top = {
let c = std::ffi::CString::new("u_top")?;
unsafe { (gl.gl_get_uniform_location)(grad_program, c.as_ptr()) }
};
let grad_uniform_bottom = {
let c = std::ffi::CString::new("u_bottom")?;
unsafe { (gl.gl_get_uniform_location)(grad_program, c.as_ptr()) }
};
// 遮罩mask程序仅用于把 mask drawable 写入 stencil
let mask_vert = compile_shader(&gl, gl::GL_VERTEX_SHADER, MASK_VERT_SHADER_SRC)?;
let mask_frag = compile_shader(&gl, gl::GL_FRAGMENT_SHADER, MASK_FRAG_SHADER_SRC)?;
let mask_program = unsafe {
let p = (gl.gl_create_program)();
(gl.gl_attach_shader)(p, mask_vert);
(gl.gl_attach_shader)(p, mask_frag);
(gl.gl_link_program)(p);
let mut status = 0;
(gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status);
if status == 0 {
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
return Err(anyhow!("mask shader link failed: {}", String::from_utf8_lossy(&log)));
}
p
};
let mask_attrib_position = {
let c = std::ffi::CString::new("a_position")?;
unsafe { (gl.gl_get_attrib_location)(mask_program, c.as_ptr()) }
};
let mask_attrib_tex_coord = {
let c = std::ffi::CString::new("a_texCoord")?;
unsafe { (gl.gl_get_attrib_location)(mask_program, c.as_ptr()) }
};
let mask_uniform_matrix = {
let c = std::ffi::CString::new("u_matrix")?;
unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) }
};
let mask_uniform_texture = {
let c = std::ffi::CString::new("s_texture0")?;
unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) }
};
let mask_uniform_multiply_color = {
let c = std::ffi::CString::new("u_multiplyColor")?;
unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) }
};
let mask_uniform_screen_color = {
let c = std::ffi::CString::new("u_screenColor")?;
unsafe { (gl.gl_get_uniform_location)(mask_program, c.as_ptr()) }
};
// 文字 overlay 程序
let text_vert = compile_shader(&gl, gl::GL_VERTEX_SHADER, TEXT_VERT_SHADER_SRC)?;
let text_frag = compile_shader(&gl, gl::GL_FRAGMENT_SHADER, TEXT_FRAG_SHADER_SRC)?;
let text_program = unsafe {
let p = (gl.gl_create_program)();
(gl.gl_attach_shader)(p, text_vert);
(gl.gl_attach_shader)(p, text_frag);
(gl.gl_link_program)(p);
let mut status = 0;
(gl.gl_get_programiv)(p, gl::GL_LINK_STATUS, &mut status);
if status == 0 {
let mut log: Vec<u8> = vec![0u8; 1024];
(gl.gl_get_program_info_log)(p, 1024, std::ptr::null_mut(), log.as_mut_ptr() as *mut c_char);
return Err(anyhow!("text shader link failed: {}", String::from_utf8_lossy(&log)));
}
p
};
let text_uniform_matrix = {
let c = std::ffi::CString::new("u_matrix")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_uniform_color = {
let c = std::ffi::CString::new("u_color")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_uniform_texture = {
let c = std::ffi::CString::new("s_texture0")?;
unsafe { (gl.gl_get_uniform_location)(text_program, c.as_ptr()) }
};
let text_attrib_position = {
let c = std::ffi::CString::new("a_position")?;
unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) }
};
let text_attrib_tex_coord = {
let c = std::ffi::CString::new("a_texCoord")?;
unsafe { (gl.gl_get_attrib_location)(text_program, c.as_ptr()) }
};
println!(
"[Live2D] GL 程序就绪: program={} pos={} tex={} mat={} tex_u={} base={} mult={} scr={}",
program, attrib_position, attrib_tex_coord, uniform_matrix,
uniform_texture, uniform_base_color, uniform_multiply_color, uniform_screen_color
);
println!(
"[Live2D] 渐变程序就绪: grad={} pos={} top={} bottom={}",
grad_program, grad_attrib_pos, grad_uniform_top, grad_uniform_bottom
);
Ok(Self {
gl,
display,
surface,
context,
x_display,
x_window,
program,
@@ -578,8 +958,25 @@ impl RenderContext {
uniform_base_color,
uniform_multiply_color,
uniform_screen_color,
width,
height,
grad_program,
grad_attrib_pos,
grad_uniform_top,
grad_uniform_bottom,
mask_program,
mask_attrib_position,
mask_attrib_tex_coord,
mask_uniform_matrix,
mask_uniform_texture,
mask_uniform_multiply_color,
mask_uniform_screen_color,
text_program,
text_uniform_matrix,
text_uniform_color,
text_uniform_texture,
text_attrib_position,
text_attrib_tex_coord,
width: win_w,
height: win_h,
})
}
@@ -617,7 +1014,70 @@ impl RenderContext {
}
}
/// 绘制全屏渐变背景在模型之前调用。top/bottom 为 RGB范围 0..1。
pub fn draw_background(&self, top: [f32; 3], bottom: [f32; 3]) {
unsafe {
// 背景不透明,先关闭混合
(self.gl.gl_disable)(gl::GL_BLEND);
(self.gl.gl_use_program)(self.grad_program);
let verts: [f32; 8] = [
-1.0, -1.0,
1.0, -1.0,
-1.0, 1.0,
1.0, 1.0,
];
let mut vbo = 0;
(self.gl.gl_gen_buffers)(1, &mut vbo);
(self.gl.gl_bind_buffer)(gl::GL_ARRAY_BUFFER, vbo);
(self.gl.gl_buffer_data)(
gl::GL_ARRAY_BUFFER,
(verts.len() * 4) as isize,
verts.as_ptr() as *const c_void,
gl::GL_STATIC_DRAW,
);
(self.gl.gl_enable_vertex_attrib_array)(self.grad_attrib_pos as u32);
(self.gl.gl_vertex_attrib_pointer)(
self.grad_attrib_pos as u32,
2,
gl::GL_FLOAT,
gl::GL_FALSE,
0,
std::ptr::null(),
);
(self.gl.gl_uniform4f)(self.grad_uniform_top, top[0], top[1], top[2], 1.0);
(self.gl.gl_uniform4f)(self.grad_uniform_bottom, bottom[0], bottom[1], bottom[2], 1.0);
(self.gl.gl_draw_arrays)(gl::GL_TRIANGLE_STRIP, 0, 4);
(self.gl.gl_delete_buffers)(1, &vbo);
// 恢复混合,供后续模型绘制
(self.gl.gl_enable)(gl::GL_BLEND);
}
}
pub fn clear(&self) {
unsafe { (self.gl.gl_clear)(gl::GL_COLOR_BUFFER_BIT); }
}
/// 将窗口提升到 X11 堆叠顺序的最顶层
pub fn raise_window(&self) {
unsafe {
(self.gl.x_raise_window)(self.x_display, self.x_window);
}
}
/// 将 EGL 上下文绑定到当前线程EGL 上下文是线程绑定的)
pub fn make_current(&self) -> Result<()> {
if unsafe { (self.gl.egl_make_current)(self.display, self.surface, self.surface, self.context) } == 0 {
return Err(anyhow!("eglMakeCurrent 失败(子线程)"));
}
Ok(())
}
/// 释放当前线程的 EGL 上下文绑定,允许其他线程绑定
pub fn release_current(&self) {
unsafe {
(self.gl.egl_make_current)(self.display, std::ptr::null_mut(), std::ptr::null_mut(), std::ptr::null_mut());
}
}
}

View File

@@ -6,6 +6,7 @@
//! - `assets`model3.json 解析、PNG 纹理加载
//! - `gl`EGL 上下文 + OpenGL ES 2.0 着色器/渲染
//! - `animation`:参数动画(眨眼、嘴部、待机呼吸)
//! - `pose`Pose PartOpacity 切换(避免互斥手臂/服装同时显示)
//! - `renderer`:整合渲染循环
pub mod animation;
@@ -13,7 +14,9 @@ pub mod assets;
pub mod core_ffi;
pub mod gl;
pub mod model;
pub mod pose;
pub mod renderer;
pub mod text;
mod plugin;
pub use plugin::Live2DPlugin;

View File

@@ -11,6 +11,8 @@ use std::path::Path;
/// 一个 drawable 的渲染所需数据快照
#[derive(Clone)]
pub struct DrawableSnapshot {
/// 在 Core drawable 数组中的原始索引(用于查 mask 引用)
pub index: usize,
pub texture_index: i32,
pub draw_order: i32,
pub render_order: i32,
@@ -23,6 +25,8 @@ pub struct DrawableSnapshot {
pub indices: Vec<u16>,
pub multiply_color: [f32; 4],
pub screen_color: [f32; 4],
/// 本 drawable 依赖的 mask drawable 索引(空表示无遮罩)
pub mask_indices: Vec<i32>,
pub mask_counts: i32,
}
@@ -60,6 +64,12 @@ pub struct CubismModel {
pub param_ids: Vec<String>,
/// 参数索引映射id → index
pub param_index: std::collections::HashMap<String, usize>,
/// Part 数量
pub part_count: usize,
/// Part ID 列表(与 Core 内部顺序一致)
pub part_ids: Vec<String>,
/// Part 索引映射id → index
pub part_index: std::collections::HashMap<String, usize>,
/// drawable 数量
pub drawable_count: usize,
/// canvas 信息
@@ -130,6 +140,20 @@ impl CubismModel {
param_ids.push(id);
}
// 读取 part 信息Pose 会按 PartOpacity 切换互斥部件,如 Haru 的两组手臂)
let part_count = unsafe { (core.csm_get_part_count)(model) } as usize;
let part_ids_ptr = unsafe { (core.csm_get_part_ids)(model) };
let mut part_ids = Vec::with_capacity(part_count);
let mut part_index = std::collections::HashMap::new();
for i in 0..part_count {
let id = unsafe {
super::core_ffi::cstr_ptr_to_string(*part_ids_ptr.add(i))
.unwrap_or_default()
};
part_index.insert(id.clone(), i);
part_ids.push(id);
}
// 读取 drawable 数量
let drawable_count = unsafe { (core.csm_get_drawable_count)(model) } as usize;
@@ -148,8 +172,8 @@ impl CubismModel {
};
println!(
"[Live2D] 模型加载成功: 参数={} drawable={} canvas={:?}x{:.0}ppu",
param_count, drawable_count, canvas.size, canvas.pixels_per_unit
"[Live2D] 模型加载成功: 参数={} part={} drawable={} canvas={:?} origin=({:.1},{:.1})x{:.0}ppu",
param_count, part_count, drawable_count, canvas.size, canvas.origin[0], canvas.origin[1], canvas.pixels_per_unit
);
Ok(Self {
@@ -161,6 +185,9 @@ impl CubismModel {
param_count,
param_ids,
param_index,
part_count,
part_ids,
part_index,
drawable_count,
canvas,
})
@@ -201,19 +228,34 @@ impl CubismModel {
/// 按参数 ID 设置值
pub fn set_param(&self, core: &CubismCore, id: &str, value: f32) -> bool {
if let Some(&idx) = self.param_index.get(id) {
let slice = self.param_values_mut(core);
slice[idx] = value;
self.set_param_by_index(core, idx, value);
true
} else {
false
}
}
/// 按参数索引设置值
pub fn set_param_by_index(&self, core: &CubismCore, index: usize, value: f32) {
if index < self.param_count {
let slice = self.param_values_mut(core);
slice[index] = value;
}
}
/// 按参数 ID 获取值
pub fn get_param(&self, core: &CubismCore, id: &str) -> Option<f32> {
let idx = *self.param_index.get(id)?;
self.get_param_by_index(core, idx)
}
/// 按参数索引获取值
pub fn get_param_by_index(&self, core: &CubismCore, index: usize) -> Option<f32> {
if index >= self.param_count {
return None;
}
let slice = self.param_values_mut(core);
Some(slice[idx])
Some(slice[index])
}
/// 将所有参数重置为默认值
@@ -225,6 +267,36 @@ impl CubismModel {
}
}
/// 获取 PartOpacity 缓冲区(可读写)
pub fn part_opacities_mut(&self, core: &CubismCore) -> &mut [f32] {
unsafe {
let p = (core.csm_get_part_opacities)(self.model);
std::slice::from_raw_parts_mut(p, self.part_count)
}
}
/// 按 Part ID 获取索引
pub fn part_index(&self, id: &str) -> Option<usize> {
self.part_index.get(id).copied()
}
/// 按 Part 索引设置 opacity
pub fn set_part_opacity_by_index(&self, core: &CubismCore, index: usize, opacity: f32) {
if index < self.part_count {
let slice = self.part_opacities_mut(core);
slice[index] = opacity.clamp(0.0, 1.0);
}
}
/// 按 Part 索引获取 opacity
pub fn get_part_opacity_by_index(&self, core: &CubismCore, index: usize) -> Option<f32> {
if index >= self.part_count {
return None;
}
let slice = self.part_opacities_mut(core);
Some(slice[index])
}
/// 更新 model应用参数变化重新计算顶点/不透明度等)
pub fn update(&self, core: &CubismCore) {
unsafe { (core.csm_update_model)(self.model) };
@@ -254,6 +326,7 @@ impl CubismModel {
let indices = (core.csm_get_drawable_indices)(self.model);
let multiply_colors = (core.csm_get_drawable_multiply_colors)(self.model);
let screen_colors = (core.csm_get_drawable_screen_colors)(self.model);
let masks = (core.csm_get_drawable_masks)(self.model);
// 先收集每个 drawable 的原始索引
let mut items: Vec<(usize, i32)> = (0..count)
@@ -287,7 +360,18 @@ impl CubismModel {
let mc = *multiply_colors.add(i);
let sc = *screen_colors.add(i);
// 取出本 drawable 的 mask 索引列表
let mk = *mask_counts.add(i) as usize;
let mut mask_indices: Vec<i32> = Vec::with_capacity(mk);
if mk > 0 {
let mptr = *masks.add(i);
for k in 0..mk {
mask_indices.push(*mptr.add(k));
}
}
DrawableSnapshot {
index: i,
texture_index: *texture_indices.add(i),
draw_order: *draw_orders.add(i),
render_order: ro,
@@ -300,7 +384,8 @@ impl CubismModel {
indices: idx_vec,
multiply_color: [mc.x, mc.y, mc.z, mc.w],
screen_color: [sc.x, sc.y, sc.z, sc.w],
mask_counts: *mask_counts.add(i),
mask_indices,
mask_counts: mk as i32,
}
})
.collect()

View File

@@ -9,10 +9,11 @@ use crate::core::plugin::{Platform, Plugin, PluginContext, PluginInfo};
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use super::renderer::Live2DRenderer;
use super::renderer::{Live2DRenderer, Subtitle, SubtitleState};
pub struct Live2DPlugin {
ctx: Option<PluginContext>,
@@ -21,6 +22,8 @@ pub struct Live2DPlugin {
worker: Option<JoinHandle<()>>,
/// 渲染停止信号
stop_flag: Arc<AtomicBool>,
/// 字幕共享状态http 插件写入,渲染线程读取)
subtitle: SubtitleState,
}
impl Live2DPlugin {
@@ -30,6 +33,7 @@ impl Live2DPlugin {
renderer: None,
worker: None,
stop_flag: Arc::new(AtomicBool::new(false)),
subtitle: Arc::new(Mutex::new(None)),
}
}
}
@@ -75,6 +79,13 @@ impl Plugin for Live2DPlugin {
message: Message::PluginReady(self.id().to_string()),
})?;
// 如果初始配置就是 Live2D 模式,立即启动渲染
let config = (*ctx.config).clone();
if config.character.render_type == RenderType::Live2d {
println!("[Live2DPlugin] 检测到初始配置为 Live2D 模式,启动渲染");
self.start_live2d(&config)?;
}
Ok(())
}
@@ -90,6 +101,16 @@ impl Plugin for Live2DPlugin {
Message::Shutdown => {
self.stop_live2d();
}
Message::Subtitle { text } => {
// 设备端字幕:写入共享状态,渲染线程每帧读取,超时自动消失
let ttl = Duration::from_millis(
(text.chars().count().max(1) as u64)
.saturating_mul(220)
.saturating_add(2500),
);
*self.subtitle.lock().unwrap() =
Some(Subtitle { text, expires_at: Instant::now() + ttl });
}
_ => {}
}
Ok(())
@@ -132,11 +153,34 @@ impl Live2DPlugin {
.clone();
let renderer = Arc::new(std::sync::Mutex::new(
Live2DRenderer::new(&core_so, &live2d_base, &model_rel, width, height)?,
Live2DRenderer::new(
&core_so,
&live2d_base,
&model_rel,
width,
height,
Arc::clone(&self.subtitle),
)?,
));
self.renderer = Some(Arc::clone(&renderer));
// 渲染器初始化(含 texture 上传)已完成,释放主线程的 EGL 上下文绑定
// 让渲染子线程能重新绑定(同一个 EGL context 不能同时在两个线程 current
{
let r = renderer.lock().unwrap();
r.gl_ctx.release_current();
}
let handle = std::thread::spawn(move || {
// EGL 上下文是线程绑定的,需在渲染线程重新绑定
{
let r = renderer.lock().unwrap();
if let Err(e) = r.gl_ctx.make_current() {
eprintln!("[Live2DPlugin] eglMakeCurrent 失败: {e}");
return;
}
}
// 渲染循环
loop {
if stop_flag.load(Ordering::SeqCst) {

233
src/plugins/live2d/pose.rs Normal file
View File

@@ -0,0 +1,233 @@
//! Pose 处理:根据 `.pose3.json` 管理互斥 Part 的显示状态。
//!
//! Haru 这类模型会把“手臂伸开”和“手臂抱胸”做成不同 Part。
//! 如果不执行 Pose多个 Part 会同时可见,画面上就会出现四只手臂。
use super::core_ffi::CubismCore;
use super::model::CubismModel;
use anyhow::{Context, Result};
use serde::Deserialize;
use std::path::Path;
const DEFAULT_FADE_IN_SECONDS: f32 = 0.5;
const EPSILON: f32 = 0.001;
const PHI: f32 = 0.5;
const BACK_OPACITY_THRESHOLD: f32 = 0.15;
#[derive(Deserialize)]
struct PoseJson {
#[serde(rename = "FadeInTime")]
fade_in_time: Option<f32>,
#[serde(rename = "Groups", default)]
groups: Vec<Vec<PosePartJson>>,
}
#[derive(Deserialize)]
struct PosePartJson {
#[serde(rename = "Id")]
id: String,
#[serde(rename = "Link", default)]
link: Vec<String>,
}
#[derive(Clone, Debug)]
struct PosePart {
part_index: Option<usize>,
param_index: Option<usize>,
link_part_indices: Vec<usize>,
virtual_param_value: f32,
}
/// Live2D Pose 状态。
pub struct PoseState {
fade_time_seconds: f32,
groups: Vec<Vec<PosePart>>,
initialized: bool,
}
impl PoseState {
pub fn from_path(path: &Path, model: &CubismModel) -> Result<Self> {
let text = std::fs::read_to_string(path)
.with_context(|| format!("读取 pose3.json 失败: {}", path.display()))?;
let json: PoseJson = serde_json::from_str(&text)
.with_context(|| format!("解析 pose3.json 失败: {}", path.display()))?;
let fade_time_seconds = json
.fade_in_time
.filter(|v| *v >= 0.0)
.unwrap_or(DEFAULT_FADE_IN_SECONDS);
let groups: Vec<Vec<PosePart>> = json
.groups
.into_iter()
.map(|group| {
group
.into_iter()
.map(|part| PosePart {
param_index: model.param_index.get(&part.id).copied(),
part_index: model.part_index(&part.id),
link_part_indices: part
.link
.iter()
.filter_map(|id| model.part_index(id))
.collect(),
virtual_param_value: 0.0,
})
.collect()
})
.collect();
println!(
"[Live2D] Pose 加载: {} groups={} fade={:.3}s",
path.display(),
groups.len(),
fade_time_seconds
);
Ok(Self {
fade_time_seconds,
groups,
initialized: false,
})
}
/// 初始化 Pose每个互斥组保留当前最可见的 Part隐藏其余 Part。
pub fn reset(&mut self, core: &CubismCore, model: &CubismModel) {
for group in &mut self.groups {
let mut visible_index = 0;
let mut max_opacity = -1.0;
for (index, part) in group.iter().enumerate() {
let opacity = part
.part_index
.and_then(|idx| model.get_part_opacity_by_index(core, idx))
.unwrap_or(0.0);
if opacity >= max_opacity {
max_opacity = opacity;
visible_index = index;
}
}
for (index, part) in group.iter_mut().enumerate() {
let opacity = if index == visible_index { 1.0 } else { 0.0 };
part.virtual_param_value = opacity;
if let Some(param_index) = part.param_index {
model.set_param_by_index(core, param_index, opacity);
}
if let Some(part_index) = part.part_index {
model.set_part_opacity_by_index(core, part_index, opacity);
}
}
}
self.copy_part_opacities(core, model);
self.initialized = true;
}
/// 按官方 CubismPose 的互斥淡入规则更新 PartOpacity。
pub fn update_parameters(&mut self, core: &CubismCore, model: &CubismModel, dt: f32) {
if !self.initialized {
self.reset(core, model);
}
let dt = dt.max(0.0);
let fade_time_seconds = self.fade_time_seconds;
for group in &mut self.groups {
Self::do_fade(core, model, fade_time_seconds, dt, group);
}
self.copy_part_opacities(core, model);
}
fn do_fade(
core: &CubismCore,
model: &CubismModel,
fade_time_seconds: f32,
dt: f32,
group: &mut [PosePart],
) {
if group.is_empty() {
return;
}
let mut visible_part_index = None;
let mut new_opacity = 1.0;
for (index, part) in group.iter().enumerate() {
if part.pose_value(core, model) <= EPSILON {
continue;
}
visible_part_index = Some(index);
if fade_time_seconds == 0.0 {
new_opacity = 1.0;
} else {
new_opacity = part
.part_index
.and_then(|idx| model.get_part_opacity_by_index(core, idx))
.unwrap_or(0.0);
new_opacity = (new_opacity + dt / fade_time_seconds).min(1.0);
}
break;
}
let visible_part_index = visible_part_index.unwrap_or(0);
if visible_part_index == 0 && group[0].pose_value(core, model) <= EPSILON {
group[0].virtual_param_value = 1.0;
if let Some(param_index) = group[0].param_index {
model.set_param_by_index(core, param_index, 1.0);
}
}
for (index, part) in group.iter().enumerate() {
let Some(part_index) = part.part_index else {
continue;
};
if index == visible_part_index {
model.set_part_opacity_by_index(core, part_index, new_opacity);
continue;
}
let mut opacity = model
.get_part_opacity_by_index(core, part_index)
.unwrap_or(0.0);
let mut target_opacity = if new_opacity < PHI {
new_opacity * (PHI - 1.0) / PHI + 1.0
} else {
(1.0 - new_opacity) * PHI / (1.0 - PHI)
};
let back_opacity = (1.0 - target_opacity) * (1.0 - new_opacity);
if back_opacity > BACK_OPACITY_THRESHOLD {
target_opacity = 1.0 - BACK_OPACITY_THRESHOLD / (1.0 - new_opacity);
}
if opacity > target_opacity {
opacity = target_opacity;
}
model.set_part_opacity_by_index(core, part_index, opacity);
}
}
fn copy_part_opacities(&self, core: &CubismCore, model: &CubismModel) {
for group in &self.groups {
for part in group {
let Some(part_index) = part.part_index else {
continue;
};
let Some(opacity) = model.get_part_opacity_by_index(core, part_index) else {
continue;
};
for &link_part_index in &part.link_part_indices {
model.set_part_opacity_by_index(core, link_part_index, opacity);
}
}
}
}
}
impl PosePart {
fn pose_value(&self, core: &CubismCore, model: &CubismModel) -> f32 {
self.param_index
.and_then(|idx| model.get_param_by_index(core, idx))
.unwrap_or(self.virtual_param_value)
}
}

View File

@@ -1,15 +1,32 @@
//! Live2D 渲染器:整å<EFBFBD>ˆ Core + Model + GL + Animation
//! Live2D Renderer: Core + Model + GL + Animation integration
use super::animation::AnimationState;
use super::assets::{self, Model3Json, TextureImage};
use super::core_ffi::{self, CubismCore};
use super::gl::{self as gl_api, RenderContext};
use gl_api::gl as gl_const;
use super::model::{CubismModel, DrawableSnapshot};
use anyhow::{Context, Result};
use std::path::Path;
use super::pose::PoseState;
use super::text::TextRenderer;
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::{Duration, Instant};
/// 前几帧打印被遮罩 drawable 的 mask 引用关系,便于排查遮罩错误(如后臂整只伸开)。
static LIVE2D_MASK_DEBUG: AtomicU32 = AtomicU32::new(0);
/// 设备端字幕AI 回复文字 + 自动消失时间。
pub struct Subtitle {
pub text: String,
pub expires_at: Instant,
}
/// 字幕共享状态http 插件写入live2d 渲染线程读取)。
pub type SubtitleState = Arc<Mutex<Option<Subtitle>>>;
pub struct Live2DRenderer {
pub core: Arc<CubismCore>,
pub model: CubismModel,
@@ -18,8 +35,13 @@ pub struct Live2DRenderer {
pub gl_ctx: RenderContext,
pub texture_ids: Vec<u32>,
pub animation: AnimationState,
pub pose: Option<PoseState>,
pub model_matrix: [f32; 16],
pub running: bool,
/// 字幕共享状态(由 live2d 插件从 http 插件接收并传入)
pub subtitle: SubtitleState,
/// 中文字幕渲染器(无可用字体时为 None
pub text: Option<TextRenderer>,
}
impl Live2DRenderer {
@@ -29,14 +51,15 @@ impl Live2DRenderer {
model3_json_rel: &str,
width: i32,
height: i32,
subtitle: SubtitleState,
) -> Result<Self> {
let core = Arc::new(CubismCore::load(core_so_path)?);
println!("[Live2D] Core 版本: {}", core.version_string());
println!("[Live2D] Core version: {}", core.version_string());
let paths = assets::resolve_asset_paths(live2d_base_dir, model3_json_rel)?;
let model3 = Model3Json::from_path(&paths.model3_json)?;
println!(
"[Live2D] model3.json: 版本={} moc={} 纹ç<EFBFBD>†æ•?{}",
"[Live2D] model3.json: ver={} moc={} textures={}",
model3.version,
model3.file_references.moc,
model3.file_references.textures.len()
@@ -48,12 +71,37 @@ impl Live2DRenderer {
let gl_ctx = RenderContext::new_fullscreen(width, height)?;
let texture_ids: Vec<u32> = textures.iter().map(|t| gl_ctx.create_texture(t)).collect();
println!("[Live2D] 已上ä¼?{} 个纹ç<C2B9>†åˆ° GPU", texture_ids.len());
println!("[Live2D] Uploaded {} textures", texture_ids.len());
let canvas = model.canvas;
let model_matrix = compute_model_matrix(canvas, width, height);
// 用真实窗口尺寸(已被 gl_ctx 修正为显示器实际分辨率)计算矩阵
let model_matrix = compute_model_matrix(canvas, gl_ctx.width, gl_ctx.height);
model.reset_to_default(&core);
let mut pose = if let Some(pose_rel) = model3.file_references.pose.as_deref() {
let pose_path = paths.model3_dir.join(pose_rel);
Some(PoseState::from_path(&pose_path, &model)?)
} else {
None
};
if let Some(pose) = pose.as_mut() {
pose.reset(&core, &model);
}
// 初始化中文字幕渲染器(找不到字体则降级为无字幕)
let text = match find_cjk_font() {
Some(p) => match TextRenderer::new(p.as_path(), 44) {
Ok(t) => Some(t),
Err(e) => {
eprintln!("[Live2D] 字幕渲染器初始化失败,字幕功能禁用: {e}");
None
}
},
None => {
eprintln!("[Live2D] 未找到中文字体,字幕功能禁用");
None
}
};
Ok(Self {
core,
@@ -63,8 +111,11 @@ impl Live2DRenderer {
gl_ctx,
texture_ids,
animation: AnimationState::default(),
pose,
model_matrix,
running: false,
subtitle,
text,
})
}
@@ -77,75 +128,129 @@ impl Live2DRenderer {
let dt = 1.0 / 60.0;
self.animation.update(&self.core, &self.model, now, dt);
if let Some(pose) = self.pose.as_mut() {
pose.update_parameters(&self.core, &self.model, dt);
}
self.model.update(&self.core);
let snapshots = self.model.drawable_snapshots(&self.core);
// 前 1 帧打印全部 drawable 的概要 + 顶点包围盒,用于排查遮罩([Live2D][maskdbg]
let dbg = LIVE2D_MASK_DEBUG.fetch_add(1, Ordering::SeqCst);
if dbg < 1 {
eprintln!("[Live2D][maskdbg] total={}", snapshots.len());
for s in &snapshots {
let (mut minx, mut miny, mut maxx, mut maxy) =
(f32::MAX, f32::MAX, f32::MIN, f32::MIN);
for v in &s.vertex_positions {
minx = minx.min(v[0]);
miny = miny.min(v[1]);
maxx = maxx.max(v[0]);
maxy = maxy.max(v[1]);
}
let inv = (s.constant_flags & super::core_ffi::constant_flags::IS_INVERTED_MASK)
!= 0;
eprintln!(
"[Live2D][maskdbg] i={} ro={} op={:.2} fl={:#04x} masked={} masks={:?} inv={} bbox=({:.0},{:.0})-({:.0},{:.0}) tex={}",
s.index,
s.render_order,
s.opacity,
s.constant_flags,
!s.mask_indices.is_empty(),
s.mask_indices,
inv,
minx,
miny,
maxx,
maxy,
s.texture_index
);
}
}
unsafe {
let gl = &self.gl_ctx.gl;
// 每帧校正 viewport 为窗口实际尺寸,避免窗口被 WM 调整大小后
// 仍用旧的小尺寸(如 800x800绘制导致模型只出现在左下角。
(gl.gl_viewport)(0, 0, self.gl_ctx.width, self.gl_ctx.height);
(gl.gl_clear_color)(0.0, 0.0, 0.0, 1.0);
(gl.gl_clear)(gl_api::GL_COLOR_BUFFER_BIT);
(gl.gl_enable)(gl_api::GL_BLEND);
(gl.gl_clear)(gl_const::GL_COLOR_BUFFER_BIT);
// 先画渐变背景再画模型draw_background 内部会恢复 BLEND
self.gl_ctx.draw_background(
[0.05, 0.09, 0.17], // 顶部:深蓝(全息风)
[0.01, 0.02, 0.04], // 底部:近黑
);
(gl.gl_enable)(gl_const::GL_BLEND);
(gl.gl_use_program)(self.gl_ctx.program);
(gl.gl_uniform_matrix4fv)(
self.gl_ctx.uniform_matrix,
1,
gl_api::GL_FALSE,
gl_const::GL_FALSE,
self.model_matrix.as_ptr(),
);
}
// 建立 原始索引 -> 快照 的查找表,供被遮罩 drawable 引用其 mask
let snap_by_index: std::collections::HashMap<usize, &DrawableSnapshot> =
snapshots.iter().map(|s| (s.index, s)).collect();
for snap in &snapshots {
if snap.opacity <= 0.001 {
continue;
}
self.draw_drawable(snap);
if snap.mask_indices.is_empty() {
self.draw_drawable(snap);
} else {
self.draw_masked_drawable(snap, &snap_by_index);
}
}
// 设备端字幕 overlayAI 回复文字)
self.draw_subtitle();
self.model.reset_dynamic_flags(&self.core);
unsafe {
(self.gl_ctx.gl.egl_swap_buffers)(self.gl_ctx.display, self.gl_ctx.surface);
}
// 每帧提升窗口到最顶层,防止桌面/其他窗口遮挡
self.gl_ctx.raise_window();
Ok(())
}
fn draw_drawable(&self, snap: &DrawableSnapshot) {
let gl = &self.gl_ctx.gl;
unsafe {
// 纹ç<C2B9>
let tex_idx = snap.texture_index as usize;
if tex_idx < self.texture_ids.len() {
(gl.gl_active_texture)(gl_api::GL_TEXTURE0);
(gl.gl_bind_texture)(gl_api::GL_TEXTURE_2D, self.texture_ids[tex_idx]);
(gl.gl_active_texture)(gl_const::GL_TEXTURE0);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, self.texture_ids[tex_idx]);
(gl.gl_uniform1i)(self.gl_ctx.uniform_texture, 0);
}
// base color(ä¸<C3A4>é€<C3A9>明度乘åˆ?alphaï¼? (gl.gl_uniform4f)(self.gl_ctx.uniform_base_color, 1.0, 1.0, 1.0, snap.opacity);
// multiply/screen color
(gl.gl_uniform4f)(self.gl_ctx.uniform_base_color, 1.0, 1.0, 1.0, snap.opacity);
let mc = snap.multiply_color;
(gl.gl_uniform4f)(self.gl_ctx.uniform_multiply_color, mc[0], mc[1], mc[2], mc[3]);
let sc = snap.screen_color;
(gl.gl_uniform4f)(self.gl_ctx.uniform_screen_color, sc[0], sc[1], sc[2], sc[3]);
// æ··å<C2B7>ˆæ¨¡å¼<C3A5>
match snap.blend_mode {
core_ffi::blend_mode::NORMAL => {
(gl.gl_blend_func)(gl_api::GL_SRC_ALPHA, gl_api::GL_ONE_MINUS_SRC_ALPHA);
(gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE_MINUS_SRC_ALPHA);
}
core_ffi::blend_mode::ADDITIVE => {
(gl.gl_blend_func)(gl_api::GL_SRC_ALPHA, gl_api::GL_ONE);
(gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE);
}
core_ffi::blend_mode::MULTIPLICATIVE => {
(gl.gl_blend_func)(gl_api::GL_ZERO, gl_api::GL_SRC_COLOR);
(gl.gl_blend_func)(gl_const::GL_ZERO, gl_const::GL_SRC_COLOR);
}
_ => {
(gl.gl_blend_func)(gl_api::GL_SRC_ALPHA, gl_api::GL_ONE_MINUS_SRC_ALPHA);
(gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE_MINUS_SRC_ALPHA);
}
}
// é¡¶ç¹æ•°æ<C2B0>®ï¼šposition(xy) + texCoord(uv) 交错
let vc = snap.vertex_positions.len();
let mut vertex_data: Vec<f32> = Vec::with_capacity(vc * 4);
for i in 0..vc {
@@ -157,22 +262,22 @@ impl Live2DRenderer {
let mut vbo = 0u32;
(gl.gl_gen_buffers)(1, &mut vbo);
(gl.gl_bind_buffer)(gl_api::GL_ARRAY_BUFFER, vbo);
(gl.gl_bind_buffer)(gl_const::GL_ARRAY_BUFFER, vbo);
(gl.gl_buffer_data)(
gl_api::GL_ARRAY_BUFFER,
gl_const::GL_ARRAY_BUFFER,
(vertex_data.len() * 4) as isize,
vertex_data.as_ptr() as *const std::ffi::c_void,
gl_api::GL_STATIC_DRAW,
gl_const::GL_STATIC_DRAW,
);
let mut ibo = 0u32;
(gl.gl_gen_buffers)(1, &mut ibo);
(gl.gl_bind_buffer)(gl_api::GL_ELEMENT_ARRAY_BUFFER, ibo);
(gl.gl_bind_buffer)(gl_const::GL_ELEMENT_ARRAY_BUFFER, ibo);
(gl.gl_buffer_data)(
gl_api::GL_ELEMENT_ARRAY_BUFFER,
gl_const::GL_ELEMENT_ARRAY_BUFFER,
(snap.indices.len() * 2) as isize,
snap.indices.as_ptr() as *const std::ffi::c_void,
gl_api::GL_STATIC_DRAW,
gl_const::GL_STATIC_DRAW,
);
let stride = 4 * 4;
@@ -181,8 +286,8 @@ impl Live2DRenderer {
(gl.gl_vertex_attrib_pointer)(
self.gl_ctx.attrib_position as u32,
2,
gl_api::GL_FLOAT,
gl_api::GL_FALSE,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
std::ptr::null(),
);
@@ -192,17 +297,17 @@ impl Live2DRenderer {
(gl.gl_vertex_attrib_pointer)(
self.gl_ctx.attrib_tex_coord as u32,
2,
gl_api::GL_FLOAT,
gl_api::GL_FALSE,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
(2 * 4) as *const std::ffi::c_void,
);
}
(gl.gl_draw_elements)(
gl_api::GL_TRIANGLES,
gl_const::GL_TRIANGLES,
snap.indices.len() as i32,
gl_api::GL_UNSIGNED_SHORT,
gl_const::GL_UNSIGNED_SHORT,
std::ptr::null(),
);
@@ -211,6 +316,258 @@ impl Live2DRenderer {
}
}
/// 绘制一个被遮罩masked的 drawable
/// 先用 mask 着色器把所有 mask drawable 写入 stencil 缓冲,再以 stencil 裁切
/// 实际 drawable从而把后臂/头发等限制在 mask 形状内、不再整块溢出。
fn draw_masked_drawable(
&self,
snap: &DrawableSnapshot,
snap_by_index: &std::collections::HashMap<usize, &DrawableSnapshot>,
) {
let gl = &self.gl_ctx.gl;
unsafe {
// 1) 用 mask 程序把 mask drawable 形状写入 stencil颜色关闭
(gl.gl_enable)(gl_const::GL_STENCIL_TEST);
(gl.gl_stencil_mask)(0xFF);
(gl.gl_clear)(gl_const::GL_STENCIL_BUFFER_BIT);
(gl.gl_color_mask)(0, 0, 0, 0);
(gl.gl_stencil_func)(gl_const::GL_ALWAYS, 1, 0xFF);
(gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_INCR);
(gl.gl_use_program)(self.gl_ctx.mask_program);
(gl.gl_uniform_matrix4fv)(
self.gl_ctx.mask_uniform_matrix,
1,
gl_const::GL_FALSE,
self.model_matrix.as_ptr(),
);
for &mi in &snap.mask_indices {
if let Some(m) = snap_by_index.get(&(mi as usize)) {
self.draw_mask_drawable(m);
}
}
// 2) 用 stencil 裁切实际 drawable
(gl.gl_color_mask)(1, 1, 1, 1);
(gl.gl_stencil_op)(gl_const::GL_KEEP, gl_const::GL_KEEP, gl_const::GL_KEEP);
let inverted =
(snap.constant_flags & super::core_ffi::constant_flags::IS_INVERTED_MASK) != 0;
if inverted {
(gl.gl_stencil_func)(gl_const::GL_EQUAL, 0, 0xFF);
} else {
(gl.gl_stencil_func)(gl_const::GL_NOTEQUAL, 0, 0xFF);
}
(gl.gl_use_program)(self.gl_ctx.program);
self.draw_drawable(snap);
// 复位,避免影响后续无遮罩 drawable
(gl.gl_disable)(gl_const::GL_STENCIL_TEST);
(gl.gl_stencil_mask)(0xFF);
}
}
/// 用 mask 着色器把一个 drawable 画进 stencil颜色关闭仅 alpha 形状)。
fn draw_mask_drawable(&self, snap: &DrawableSnapshot) {
let gl = &self.gl_ctx.gl;
unsafe {
let tex_idx = snap.texture_index as usize;
if tex_idx < self.texture_ids.len() {
(gl.gl_active_texture)(gl_const::GL_TEXTURE0);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, self.texture_ids[tex_idx]);
(gl.gl_uniform1i)(self.gl_ctx.mask_uniform_texture, 0);
}
(gl.gl_uniform4f)(
self.gl_ctx.mask_uniform_multiply_color,
snap.multiply_color[0],
snap.multiply_color[1],
snap.multiply_color[2],
snap.multiply_color[3],
);
(gl.gl_uniform4f)(
self.gl_ctx.mask_uniform_screen_color,
snap.screen_color[0],
snap.screen_color[1],
snap.screen_color[2],
snap.screen_color[3],
);
let vc = snap.vertex_positions.len();
let mut vertex_data: Vec<f32> = Vec::with_capacity(vc * 4);
for i in 0..vc {
vertex_data.push(snap.vertex_positions[i][0]);
vertex_data.push(snap.vertex_positions[i][1]);
vertex_data.push(snap.vertex_uvs[i][0]);
vertex_data.push(snap.vertex_uvs[i][1]);
}
let mut vbo = 0u32;
(gl.gl_gen_buffers)(1, &mut vbo);
(gl.gl_bind_buffer)(gl_const::GL_ARRAY_BUFFER, vbo);
(gl.gl_buffer_data)(
gl_const::GL_ARRAY_BUFFER,
(vertex_data.len() * 4) as isize,
vertex_data.as_ptr() as *const std::ffi::c_void,
gl_const::GL_STATIC_DRAW,
);
let mut ibo = 0u32;
(gl.gl_gen_buffers)(1, &mut ibo);
(gl.gl_bind_buffer)(gl_const::GL_ELEMENT_ARRAY_BUFFER, ibo);
(gl.gl_buffer_data)(
gl_const::GL_ELEMENT_ARRAY_BUFFER,
(snap.indices.len() * 2) as isize,
snap.indices.as_ptr() as *const std::ffi::c_void,
gl_const::GL_STATIC_DRAW,
);
let stride = 4 * 4;
if self.gl_ctx.mask_attrib_position >= 0 {
(gl.gl_enable_vertex_attrib_array)(self.gl_ctx.mask_attrib_position as u32);
(gl.gl_vertex_attrib_pointer)(
self.gl_ctx.mask_attrib_position as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
std::ptr::null(),
);
}
if self.gl_ctx.mask_attrib_tex_coord >= 0 {
(gl.gl_enable_vertex_attrib_array)(self.gl_ctx.mask_attrib_tex_coord as u32);
(gl.gl_vertex_attrib_pointer)(
self.gl_ctx.mask_attrib_tex_coord as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
(2 * 4) as *const std::ffi::c_void,
);
}
(gl.gl_draw_elements)(
gl_const::GL_TRIANGLES,
snap.indices.len() as i32,
gl_const::GL_UNSIGNED_SHORT,
std::ptr::null(),
);
(gl.gl_delete_buffers)(1, &vbo);
(gl.gl_delete_buffers)(1, &ibo);
}
}
/// 绘制设备端字幕:屏幕底部半透明背景框 + 白色中文AI 回复文字)。
fn draw_subtitle(&mut self) {
let (text, expired) = {
let guard = self.subtitle.lock().unwrap();
match guard.as_ref() {
Some(s) => (s.text.clone(), s.expires_at <= Instant::now()),
None => return,
}
};
if expired || text.trim().is_empty() {
return;
}
let text_renderer = match self.text.as_mut() {
Some(t) => t,
None => return,
};
let gl = &self.gl_ctx.gl;
// 复制 text program 的 uniform/attrib 位置到局部,避免与 &mut text_renderer 借位冲突
let prog = self.gl_ctx.text_program;
let u_mat = self.gl_ctx.text_uniform_matrix;
let u_col = self.gl_ctx.text_uniform_color;
let u_tex = self.gl_ctx.text_uniform_texture;
let a_pos = self.gl_ctx.text_attrib_position;
let a_tco = self.gl_ctx.text_attrib_tex_coord;
unsafe {
(gl.gl_enable)(gl_const::GL_BLEND);
(gl.gl_blend_func)(gl_const::GL_SRC_ALPHA, gl_const::GL_ONE_MINUS_SRC_ALPHA);
(gl.gl_use_program)(prog);
let w = self.gl_ctx.width as f32;
let h = self.gl_ctx.height as f32;
// 像素坐标原点左上y 向下)-> 裁剪空间
let ortho: [f32; 16] = [
2.0 / w, 0.0, 0.0, 0.0,
0.0, -2.0 / h, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
-1.0, 1.0, 0.0, 1.0,
];
(gl.gl_uniform_matrix4fv)(u_mat, 1, gl_const::GL_FALSE, ortho.as_ptr());
}
let font_px = text_renderer.font_size() as f32;
let line_height = (font_px * 1.35) as i32;
let margin = 40i32;
let max_width = self.gl_ctx.width - margin * 2;
let padding = 18i32;
// 按宽度换行CJK 任意位置可断,遇 \n 强制断)
let mut lines: Vec<String> = Vec::new();
let mut cur = String::new();
let mut cur_w = 0i32;
for ch in text.chars() {
if ch == '\n' {
lines.push(std::mem::take(&mut cur));
cur_w = 0;
continue;
}
let aw = text_renderer.measure_advance(ch);
if cur_w + aw > max_width && !cur.is_empty() {
lines.push(std::mem::take(&mut cur));
cur_w = 0;
}
cur.push(ch);
cur_w += aw;
}
if !cur.is_empty() {
lines.push(cur);
}
if lines.is_empty() {
return;
}
let total_text_h = lines.len() as i32 * line_height;
let box_h = total_text_h + padding * 2;
let box_y = self.gl_ctx.height - margin - box_h;
let box_x = margin;
let box_w = self.gl_ctx.width - margin * 2;
// 背景框
let white = text_renderer.white_texture(gl);
draw_text_quad(
gl, white, box_x as f32, box_y as f32, box_w as f32, box_h as f32, [0.0, 0.0, 0.0, 0.55],
u_tex, u_col, a_pos, a_tco,
);
// 逐行绘制文字(居中)
for (li, line) in lines.iter().enumerate() {
let line_w: i32 = line.chars().map(|c| text_renderer.measure_advance(c)).sum();
let start_x = ((self.gl_ctx.width - line_w) / 2) as f32;
let baseline_y =
(box_y + padding + li as i32 * line_height + (line_height as f32 * 0.78) as i32) as f32;
let mut pen_x = start_x;
for ch in line.chars() {
if let Some(g) = text_renderer.glyph(gl, ch) {
if g.tex != 0 && g.w > 0 && g.h > 0 {
let gx = pen_x + g.left as f32;
let gy = baseline_y - g.top as f32;
draw_text_quad(
gl, g.tex, gx, gy, g.w as f32, g.h as f32, [1.0, 1.0, 1.0, 1.0],
u_tex, u_col, a_pos, a_tco,
);
}
pen_x += (g.advance >> 6) as f32;
} else {
pen_x += font_px * 0.5;
}
}
}
}
pub fn run_loop(&mut self) -> Result<()> {
self.running = true;
let frame_duration = Duration::from_millis(16);
@@ -218,7 +575,7 @@ impl Live2DRenderer {
while self.running {
let start = Instant::now();
if let Err(e) = self.render_frame() {
eprintln!("[Live2D] 渲染帧失� {e}");
eprintln!("[Live2D] render frame failed: {e}");
break;
}
let elapsed = start.elapsed();
@@ -234,29 +591,118 @@ impl Live2DRenderer {
}
}
/// 计算 model â†?NDC å<EFBFBD>˜æ<EFBFBD>¢çŸ©é˜µï¼ˆåˆ—优先 4x4ï¼?///
/// Cubism å<><C3A5>标:原ç¹åœ¨ canvas 中心,Y è½´å<C2B4>上,å<C592>•ä½<C3A4>为åƒ<C3A5>ç´ ã€?/// 需è¦<C3A8>等比缩放使模åžå®Œæ•´æ˜¾ç¤ºåœ¨çª—å<E28094>£å†…ã€?fn compute_model_matrix(
/// Compute model -> NDC transform matrix (column-major 4x4).
///
/// `csmGetDrawableVertexPositions` 返回的顶点位于**模型单位**坐标系,范围约为
/// ±(canvas_w/2 / ppu)。`pixels_per_unit`(ppu) 指明 1 模型单位 = ppu 个画布像素。
/// 因此顶点必须先乘 `ppu` 回到画布像素,再映射到 [-1,1] 裁剪空间。
///
/// 实测 Haru 的 `origin = (canvas_w/2, canvas_h/2)`,即模型坐标原点本身就位于
/// 画布中心,所以**无需再做原点平移**平移反而会把模型推出屏幕。Cubism 顶点
/// 以画布中心为原点、Y 轴向上,与裁剪空间一致,故无需平移或翻转。
///
/// 适配策略:**contain整只角色恰好放入视口**,单一均匀缩放即可。
/// NDC 立方体由视口自动拉伸到窗口,因此**绝对不能**对 x/y 轴分别乘不同系数,
/// 否则会破坏长宽比(手臂被拉变形)。这里只按画布较长边归一,使整只角色完整
/// 可见、不裁切、不拉伸;窗口为横屏时角色会填满高度、两侧留渐变背景。
fn compute_model_matrix(
canvas: super::model::CanvasInfo,
_width: i32,
_height: i32,
) -> [f32; 16] {
let canvas_w = canvas.size[0].max(1.0);
let canvas_h = canvas.size[1].max(1.0);
let cw = canvas.size[0].max(1.0);
let ch = canvas.size[1].max(1.0);
let ppu = canvas.pixels_per_unit.max(1.0);
// 计算缩放:让模åžç­‰æ¯”é€åº”窗å<EFBFBD>£
let scale_x = 2.0 / canvas_w;
let scale_y = 2.0 / canvas_h;
let scale = scale_x.min(scale_y);
// 均匀缩放按画布较长边把整只角色映射到裁剪空间margin=1.0 即恰好贴满,
// 若想留一点边距可改小,如 0.95)。
let margin = 1.0;
let u = 2.0 * ppu * margin / cw.max(ch);
// 列优先矩阵:
// [ scale 0 0 0 ]
// [ 0 scale 0 0 ]
// [ 0 0 1 0 ]
// [ 0 0 0 1 ]
[
scale, 0.0, 0.0, 0.0,
0.0, scale, 0.0, 0.0,
u, 0.0, 0.0, 0.0,
0.0, u, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
]
}
/// 用 text program 画一个带纹理的四边形TRIANGLE_STRIP
///
/// `color` 为 RGBA着色器输出 = u_color.rgb 配合字形纹理的 alpha因此背景框可传
/// 纯色 + alpha字形可传白色 + alpha=1字形纹理本身含 alpha 覆盖)。
fn draw_text_quad(
gl: &gl_api::Gl,
tex: gl_const::GLuint,
x: f32,
y: f32,
w: f32,
h: f32,
color: [f32; 4],
uniform_texture: gl_const::GLint,
uniform_color: gl_const::GLint,
attrib_position: gl_const::GLint,
attrib_tex_coord: gl_const::GLint,
) {
unsafe {
(gl.gl_active_texture)(gl_const::GL_TEXTURE0);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_uniform1i)(uniform_texture, 0);
(gl.gl_uniform4f)(uniform_color, color[0], color[1], color[2], color[3]);
let verts: [f32; 16] = [
x, y, 0.0, 0.0,
x + w, y, 1.0, 0.0,
x, y + h, 0.0, 1.0,
x + w, y + h, 1.0, 1.0,
];
let mut vbo = 0u32;
(gl.gl_gen_buffers)(1, &mut vbo);
(gl.gl_bind_buffer)(gl_const::GL_ARRAY_BUFFER, vbo);
(gl.gl_buffer_data)(
gl_const::GL_ARRAY_BUFFER,
(verts.len() * 4) as isize,
verts.as_ptr() as *const std::ffi::c_void,
gl_const::GL_STATIC_DRAW,
);
let stride = 4 * 4;
if attrib_position >= 0 {
(gl.gl_enable_vertex_attrib_array)(attrib_position as u32);
(gl.gl_vertex_attrib_pointer)(
attrib_position as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
std::ptr::null(),
);
}
if attrib_tex_coord >= 0 {
(gl.gl_enable_vertex_attrib_array)(attrib_tex_coord as u32);
(gl.gl_vertex_attrib_pointer)(
attrib_tex_coord as u32,
2,
gl_const::GL_FLOAT,
gl_const::GL_FALSE,
stride,
(2 * 4) as *const std::ffi::c_void,
);
}
(gl.gl_draw_arrays)(gl_const::GL_TRIANGLE_STRIP, 0, 4);
(gl.gl_delete_buffers)(1, &vbo);
}
}
/// 在设备中查找可用的中文字体(优先 Noto Sans/Serif CJK
fn find_cjk_font() -> Option<PathBuf> {
let candidates = [
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc",
"/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc",
"/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
];
candidates
.iter()
.map(Path::new)
.find(|p| p.exists())
.map(|p| p.to_path_buf())
}

239
src/plugins/live2d/text.rs Normal file
View File

@@ -0,0 +1,239 @@
//! 基于 FreeType 的中文文字光栅化 + OpenGL 纹理渲染(设备端 Live2D 字幕 overlay
//!
//! 设计:每个字符首次出现时用 FreeType 渲染为灰度位图(经 `render_glyph` 拿到位图),转成
//! 白色字形alpha=灰度)的 RGBA 纹理并缓存;绘制时按 pen 位置贴纹理即可。纹理创建需要
//! `Gl` 函数指针(与渲染器共用同一 EGL 上下文)。
use super::gl::gl as gl_const;
use super::gl::Gl;
use freetype::face::LoadFlag;
use freetype::Face;
use freetype::Library;
use freetype::RenderMode;
use std::collections::HashMap;
use std::path::Path;
use anyhow::Result;
/// 单个字符的缓存字形
#[derive(Clone)]
pub struct CachedGlyph {
/// 字形纹理0 表示该字符无可见像素,仅用于步进)
pub tex: gl_const::GLuint,
pub w: i32,
pub h: i32,
/// 字形位图相对 pen 原点的偏移
pub left: i32,
pub top: i32,
/// 26.6 定点像素步进(渲染端需 >> 6 转为像素)
pub advance: i32,
}
pub struct TextRenderer {
face: Face<'static>,
font_px: u32,
glyphs: HashMap<char, CachedGlyph>,
white_tex: Option<gl_const::GLuint>,
}
// FreeType 的 Face 包裹裸指针,仅在本插件单一渲染线程使用。
unsafe impl Send for TextRenderer {}
/// FT_LOAD_RENDER 标志位0x4加载字符的同时渲染为位图
const FT_LOAD_RENDER: i32 = 0x4;
impl TextRenderer {
pub fn new(font_path: &Path, font_px: u32) -> Result<Self> {
let library = Library::init()?;
// 让 Library 成为 'static程序级单例从而 Face<'static> 可长期持有而不会悬垂
let library: &'static Library = Box::leak(Box::new(library));
let face = library.new_face(font_path, 0)?;
let tr = Self {
face,
font_px,
glyphs: HashMap::new(),
white_tex: None,
};
tr.face.set_pixel_sizes(0, font_px)?;
println!("[Live2D] 字幕字体就绪: {:?} ({}px)", font_path, font_px);
Ok(tr)
}
/// 当前字号(像素)
pub fn font_size(&self) -> u32 {
self.font_px
}
/// 1x1 白色纹理用于纯色背景框alpha 恒为 1
fn white_tex(&mut self, gl: &Gl) -> gl_const::GLuint {
if let Some(t) = self.white_tex {
return t;
}
let mut tex = 0u32;
unsafe {
(gl.gl_gen_textures)(1, &mut tex);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MIN_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MAG_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_S,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_T,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
let px: [u8; 4] = [255, 255, 255, 255];
(gl.gl_tex_image_2d)(
gl_const::GL_TEXTURE_2D,
0,
gl_const::GL_RGBA as i32,
1,
1,
0,
gl_const::GL_RGBA,
gl_const::GL_UNSIGNED_BYTE,
px.as_ptr() as *const std::ffi::c_void,
);
}
self.white_tex = Some(tex);
tex
}
/// 取 1x1 白纹理(供外部画背景框)。
pub fn white_texture(&mut self, gl: &Gl) -> gl_const::GLuint {
self.white_tex(gl)
}
/// 估算字符像素宽度(用于换行布局,不创建纹理)。返回像素值。
pub fn measure_advance(&mut self, ch: char) -> i32 {
if self
.face
.load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER))
.is_ok()
{
(self.face.glyph().advance().x >> 6) as i32
} else {
(self.font_px as f32 * 0.5) as i32
}
}
fn rasterize(&mut self, gl: &Gl, ch: char) -> Option<CachedGlyph> {
let half_advance_26_6 = (self.font_px as i32) << 5; // 半字宽26.6 定点
if self
.face
.load_char(ch as usize, LoadFlag::from_bits_truncate(FT_LOAD_RENDER))
.is_err()
{
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left: 0,
top: 0,
advance: half_advance_26_6,
});
}
let glyph = self.face.glyph();
if glyph.render_glyph(RenderMode::Normal).is_err() {
let adv = glyph.advance().x as i32; // 26.6 定点
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left: 0,
top: 0,
advance: adv,
});
}
let bitmap = glyph.bitmap();
let w = bitmap.width();
let h = bitmap.rows();
let left = glyph.bitmap_left();
let top = glyph.bitmap_top();
let advance = glyph.advance().x as i32; // 26.6 定点
if w == 0 || h == 0 {
return Some(CachedGlyph {
tex: 0,
w: 0,
h: 0,
left,
top,
advance,
});
}
// 灰度 -> RGBA白色字形alpha=灰度
let buf = bitmap.buffer();
let mut rgba: Vec<u8> = Vec::with_capacity((w * h * 4) as usize);
for &g in buf {
rgba.push(255);
rgba.push(255);
rgba.push(255);
rgba.push(g);
}
let mut tex = 0u32;
unsafe {
(gl.gl_gen_textures)(1, &mut tex);
(gl.gl_bind_texture)(gl_const::GL_TEXTURE_2D, tex);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MIN_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_MAG_FILTER,
gl_const::GL_LINEAR as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_S,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_parameteri)(
gl_const::GL_TEXTURE_2D,
gl_const::GL_TEXTURE_WRAP_T,
gl_const::GL_CLAMP_TO_EDGE as i32,
);
(gl.gl_tex_image_2d)(
gl_const::GL_TEXTURE_2D,
0,
gl_const::GL_RGBA as i32,
w,
h,
0,
gl_const::GL_RGBA,
gl_const::GL_UNSIGNED_BYTE,
rgba.as_ptr() as *const std::ffi::c_void,
);
}
Some(CachedGlyph {
tex,
w,
h,
left,
top,
advance,
})
}
/// 获取(必要时光栅化并缓存)字符字形。
pub fn glyph(&mut self, gl: &Gl, ch: char) -> Option<CachedGlyph> {
if !self.glyphs.contains_key(&ch) {
if let Some(g) = self.rasterize(gl, ch) {
self.glyphs.insert(ch, g);
}
}
self.glyphs.get(&ch).cloned()
}
}

View File

@@ -118,6 +118,17 @@ impl Plugin for VideoPlugin {
.as_ref()
.context("video plugin context is not initialized")?;
// 如果是 Live2D 模式,不启动视频播放
if ctx.config.character.render_type == crate::core::config::RenderType::Live2d {
println!("[VideoPlugin] Live2D 模式,跳过视频播放启动");
ctx.tx.send(Envelope {
from: self.id().to_string(),
to: Destination::Manager,
message: Message::PluginReady(self.id().to_string()),
})?;
return Ok(());
}
let processor = Arc::new(Mutex::new(VideoProcessor::new((*ctx.config).clone())?));
let worker_processor = Arc::clone(&processor);
let tx = ctx.tx.clone();