W18: context cleanup + CLI fixes + loader audit + CI matrix (W18.1-W18.4)

- W18.1 (王测+林深): Remove g_max_tokens dead API, UTF-8 bounds protection, deduplicate token counting, 0xC0/0xC1 handling, add 13 test blocks (36 checks) - W18.2 (赵码+朱晴): Fix /context no-session error message, /status 3-state connection display - W18.3 (曹武+徐磊): plugin_loader security audit — 9 dimensions, rating C, 1 HIGH + 2 MEDIUM findings - W18.4 (马奔+胡桐): CI dual-platform matrix (Ubuntu clang-18 + Windows clang-cl), ccache, build timing baseline Build 0 error, ctest 5/5 pass, metadata check clean. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-27 19:09:21 +08:00
parent 852e2cac08
commit c545d16120
18 changed files with 945 additions and 77 deletions
--- a/plugins/context/src/context_plugin.cpp
+++ b/plugins/context/src/context_plugin.cpp
@@ -9,6 +9,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdio>
+#include <cstring>
 #include <exception>
 #include <string>
 #include <vector>
@@ -19,54 +20,101 @@

 static const dstalk_host_api_t* g_host = nullptr;
 static const dstalk_session_service_t* g_session = nullptr;
-static size_t g_max_tokens = 4096;

 // ============================================================
-// 内部 C++ 辅助：token 计数
+// 内部 C++ 辅助：共享 UTF-8 token 计数
+// W18.1: 合并 count_tokens_one_message / count_tokens_trim 的重复逻辑 (F-11.1-5)
+//        添加 UTF-8 越界保护 (F-11.1-4) 和 0xC0/0xC1 过短编码检测 (F-11.1-6)
 // ============================================================

-static bool cjk_is_ascii(unsigned char c) { return c < 0x80; }
-
-static bool cjk_starts_cjk(unsigned char c) {
-    // U+4E00-U+9FFF 在 UTF-8 中编码为 0xE4-0xE9 开头的三字节
-    return c >= 0xE4 && c <= 0xE9;
-}
-
-static size_t count_tokens_one_message(const dstalk_message_t& msg) {
-    const char* text = msg.content;
-    if (!text) return 4; // 只有 overhead
+// 统计 UTF-8 字节序列 [text, text+len) 的估算 token 数。
+// overhead: 每条消息的固定开销 token（role + separators = 4）
+// 多字节序列在越界或无效后继字节时回退为单字节 other_chars 计数，不崩溃。
+static size_t count_tokens_utf8(const char* text, size_t len, size_t overhead) {
+    if (!text || len == 0) return overhead;

    size_t ascii_chars   = 0;
    size_t chinese_chars = 0;
    size_t other_chars   = 0;

    size_t i = 0;
-    while (text[i] != '\0') {
+    while (i < len && text[i] != '\0') {
        unsigned char c = static_cast<unsigned char>(text[i]);

-        if (cjk_is_ascii(c)) {
+        if (c < 0x80) {
+            // ASCII
            ascii_chars++;
            i += 1;
-        } else if (cjk_starts_cjk(c)) {
-            chinese_chars++;
-            i += 3;
-        } else if (c >= 0xC0 && c < 0xE0) {
+        } else if (c >= 0xE4 && c <= 0xE9) {
+            // CJK Unified Ideographs (U+4E00-U+9FFF): 3-byte UTF-8 0xE4-0xE9
+            // W18.1 (F-11.1-4): 检查后续 2 字节是否在有效范围内
+            if (i + 2 >= len ||
+                (static_cast<unsigned char>(text[i + 1]) & 0xC0) != 0x80 ||
+                (static_cast<unsigned char>(text[i + 2]) & 0xC0) != 0x80) {
+                other_chars++;
+                i += 1;
+            } else {
+                chinese_chars++;
+                i += 3;
+            }
+        } else if (c >= 0xC2 && c < 0xE0) {
+            // 2-byte sequence (valid range 0xC2-0xDF)
+            // W18.1 (F-11.1-4): 检查后续 1 字节
+            if (i + 1 >= len ||
+                (static_cast<unsigned char>(text[i + 1]) & 0xC0) != 0x80) {
+                other_chars++;
+                i += 1;
+            } else {
+                other_chars++;
+                i += 2;
+            }
+        } else if (c == 0xC0 || c == 0xC1) {
+            // W18.1 (F-11.1-6): 过短编码 (overlong encoding)，非法 UTF-8 起始字节
+            // 0xC0/0xC1 永远不会出现在合法 UTF-8 中；视为单字节计入 other_chars
            other_chars++;
-            i += 2;
+            i += 1;
        } else if (c >= 0xE0 && c < 0xF0) {
-            other_chars++;
-            i += 3;
+            // Non-CJK 3-byte sequence (0xE0-0xE3, 0xEA-0xEF)
+            // CJK 范围 0xE4-0xE9 已在上方分支处理
+            if (i + 2 >= len ||
+                (static_cast<unsigned char>(text[i + 1]) & 0xC0) != 0x80 ||
+                (static_cast<unsigned char>(text[i + 2]) & 0xC0) != 0x80) {
+                other_chars++;
+                i += 1;
+            } else {
+                other_chars++;
+                i += 3;
+            }
        } else if (c >= 0xF0 && c < 0xF8) {
-            other_chars++;
-            i += 4;
+            // 4-byte sequence
+            if (i + 3 >= len ||
+                (static_cast<unsigned char>(text[i + 1]) & 0xC0) != 0x80 ||
+                (static_cast<unsigned char>(text[i + 2]) & 0xC0) != 0x80 ||
+                (static_cast<unsigned char>(text[i + 3]) & 0xC0) != 0x80) {
+                other_chars++;
+                i += 1;
+            } else {
+                other_chars++;
+                i += 4;
+            }
        } else {
+            // Continuation bytes (0x80-0xBF) and other invalid start bytes (0xF8-0xFF)
            other_chars++;
            i += 1;
        }
    }

-    size_t content_tokens = (ascii_chars / 4) + (chinese_chars / 2) + (other_chars / 3);
-    return content_tokens + 4; // +4 条消息开销 (role + separators)
+    return (ascii_chars / 4) + (chinese_chars / 2) + (other_chars / 3) + overhead;
+}
+
+// ============================================================
+// 消息级 token 计数（供 count_tokens_all 和 trim_impl 调用的薄封装）
+// ============================================================
+
+static size_t count_tokens_one_message(const dstalk_message_t& msg) {
+    const char* text = msg.content;
+    if (!text) return 4; // 只有 overhead
+    return count_tokens_utf8(text, std::strlen(text), 4);
 }

 static size_t count_tokens_all(const dstalk_message_t* msgs, int count) {
@@ -91,19 +139,7 @@ struct TrimMessage {

 static size_t count_tokens_trim(const TrimMessage& msg) {
    if (msg.content.empty()) return 4;
-    const std::string& text = msg.content;
-    size_t ascii_chars = 0, chinese_chars = 0, other_chars = 0;
-    size_t i = 0;
-    while (i < text.size()) {
-        unsigned char c = static_cast<unsigned char>(text[i]);
-        if (cjk_is_ascii(c))      { ascii_chars++;   i += 1; }
-        else if (cjk_starts_cjk(c)) { chinese_chars++; i += 3; }
-        else if (c >= 0xC0 && c < 0xE0) { other_chars++; i += 2; }
-        else if (c >= 0xE0 && c < 0xF0) { other_chars++; i += 3; }
-        else if (c >= 0xF0 && c < 0xF8) { other_chars++; i += 4; }
-        else                      { other_chars++;   i += 1; }
-    }
-    return (ascii_chars / 4) + (chinese_chars / 2) + (other_chars / 3) + 4;
+    return count_tokens_utf8(msg.content.c_str(), msg.content.size(), 4);
 }

 static size_t count_tokens_trim_vec(const std::vector<TrimMessage>& msgs) {
@@ -155,8 +191,9 @@ static int trim_impl(const dstalk_message_t* in, int in_count,
    try {
        if (!in || in_count <= 0 || !out || !out_count) return -1;

-        // W12.1: 调用方传 0 时使用 g_max_tokens 作为默认限制
-        if (max_tokens == 0) max_tokens = g_max_tokens;
+        // W18.1 (F-11.1-3): g_max_tokens 已移除，调用方必须提供有效 max_tokens；
+        // 传 0 时使用硬编码默认值 4096。
+        if (max_tokens == 0) max_tokens = 4096;

        // 将 C 数组转换为内部 vector
        std::vector<TrimMessage> messages;
@@ -249,9 +286,9 @@ static int trim_impl(const dstalk_message_t* in, int in_count,
            }
        }

-        // W12.1: 消息数量上限粗略估算（每消息 ~100 token），利用 g_max_tokens 防止消息泛滥
+        // W18.1 (F-11.1-3): 消息数量上限粗略估算（每消息 ~100 token），使用当前 max_tokens
        {
-            size_t max_msg_count = (g_max_tokens + 99) / 100;  // ceil(g_max_tokens / 100)
+            size_t max_msg_count = (max_tokens + 99) / 100;  // ceil(max_tokens / 100)
            if (max_msg_count < 1) max_msg_count = 1;
            while (non_system_msgs.size() > max_msg_count) {
                non_system_msgs.erase(non_system_msgs.begin());
@@ -281,7 +318,7 @@ static int trim_impl(const dstalk_message_t* in, int in_count,

        return 0;
    } catch (const std::exception& e) {
-        // W12.1: 防止 std::bad_alloc 等 C++ 异常穿越 C ABI 边界 → std::terminate()
+        // W12.1: 防止 std::bad_alloc 等 C++ 异常穿越 C ABI 边界 -> std::terminate()
        if (g_host) g_host->log(DSTALK_LOG_ERROR, "[context] trim_impl exception: %s", e.what());
        return -1;
    } catch (...) {
@@ -294,7 +331,7 @@ static int trim_impl(const dstalk_message_t* in, int in_count,
 // Context 服务 vtable 实现
 // ============================================================

-// W12.1: 包裹 try/catch 防止异常穿越 C ABI 边界 → std::terminate()
+// W12.1: 包裹 try/catch 防止异常穿越 C ABI 边界 -> std::terminate()
 static size_t context_count_tokens(const dstalk_message_t* msgs, int count) {
    try {
        if (!msgs || count <= 0) return 0;
@@ -315,21 +352,12 @@ static int context_trim(const dstalk_message_t* in, int in_count,
    }
 }

-// W16.2: 包裹 try/catch 防止异常穿越 C ABI 边界 (§8.3 void 仅 log)
-static void context_set_max_tokens(size_t max) {
-    try {
-        g_max_tokens = max;
-    } catch (const std::exception& e) {
-        if (g_host) g_host->log(DSTALK_LOG_ERROR, "[plugin-context] context_set_max_tokens: %s", e.what());
-    } catch (...) {
-        if (g_host) g_host->log(DSTALK_LOG_ERROR, "[plugin-context] context_set_max_tokens: unknown exception");
-    }
-}
-
+// W18.1 (F-11.1-3): g_max_tokens / context_set_max_tokens 已移除。
+// max_tokens 由调用方通过 trim() 的 max_tokens 参数直接传入；
+// 传 0 时 trim_impl 使用硬编码默认值 4096。
 static dstalk_context_service_t g_context_service = {
    context_count_tokens,
-    context_trim,
-    context_set_max_tokens
+    context_trim
 };

 // ============================================================
@@ -359,7 +387,7 @@ static int on_init(const dstalk_host_api_t* host) {
    }
 }

-// W16.2: 包裹 try/catch 防止异常穿越 C ABI 边界 — void 函数仅 log
+// W16.2: 包裹 try/catch 防止异常穿越 C ABI 边界 -- void 函数仅 log
 static void on_shutdown() {
    try {
        g_session = nullptr;