feat: add scripts for fetching and training educational corpus
- Implemented `fetch_corpus.py` to scrape Chinese educational content from Wikipedia, covering subjects like Chinese, Math, English, Chemistry, Politics, History, Geography, Physics, and Biology. - Developed `fetch_corpus_incremental.py` for incremental fetching of missing entries, with automatic retries for rate limiting and support for resuming interrupted downloads. - Created `train_chat.py` for training a dialogue model, including corpus cleaning, vocabulary management, and staged training with adaptive learning rates.
This commit is contained in:
@@ -19,7 +19,7 @@ from .baselines import FlatBaseline
|
||||
from .task import ContinuousSequenceTask
|
||||
from .trainer import Trainer
|
||||
|
||||
from .language_data import CharTokenizer, CharDataset, load_shakespeare
|
||||
from .language_data import CharTokenizer, CharDataset, load_shakespeare, load_chinese_corpus, load_textbook_corpus, load_corpus
|
||||
from .language_model import (
|
||||
LanguageConfig,
|
||||
JSpaceLanguageModel,
|
||||
@@ -88,6 +88,7 @@ __all__ = [
|
||||
"ContinuousSequenceTask", "Trainer",
|
||||
# 语言建模
|
||||
"CharTokenizer", "CharDataset", "load_shakespeare",
|
||||
"load_chinese_corpus", "load_textbook_corpus", "load_corpus",
|
||||
"LanguageConfig", "JSpaceLanguageModel",
|
||||
"ExperienceReplay", "EWCOptimizer", "ExpertPlasticity",
|
||||
# J-lens 可解释性
|
||||
|
||||
Reference in New Issue
Block a user