feat: 添加儿童级对话训练功能,优化生成逻辑,新增相关数据处理模块和测试用例

This commit is contained in:
2026-07-08 12:54:29 +08:00
parent 04d59219cb
commit 49a27e124d
6 changed files with 445 additions and 29 deletions

View File

@@ -17,6 +17,7 @@ import os
from jspaceai import (
LanguageConfig, JSpaceLanguageModel,
CharTokenizer, load_chinese_corpus,
build_child_chat_corpus,
LanguageTrainingConfig, LanguageTrainingSession,
)
@@ -138,7 +139,7 @@ def clean_corpus(max_source_mb: int | None = None, use_cache: bool = True) -> st
bilingual.append(simplified)
bilingual.append(traditional)
result = '\n\n'.join(bilingual)
result = build_child_chat_corpus(repeats=8) + '\n\n' + '\n\n'.join(bilingual)
if use_cache:
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_text(result, encoding="utf-8")