feat: add scripts for fetching and training educational corpus
- Implemented `fetch_corpus.py` to scrape Chinese educational content from Wikipedia, covering subjects like Chinese, Math, English, Chemistry, Politics, History, Geography, Physics, and Biology. - Developed `fetch_corpus_incremental.py` for incremental fetching of missing entries, with automatic retries for rate limiting and support for resuming interrupted downloads. - Created `train_chat.py` for training a dialogue model, including corpus cleaning, vocabulary management, and staged training with adaptive learning rates.
This commit is contained in:
419
scripts/build_textbook_corpus.py
Normal file
419
scripts/build_textbook_corpus.py
Normal file
@@ -0,0 +1,419 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从电子课本网（dzkbw.com）抓取义务教育阶段教材目录，并按阶段/年级/科目/版本建立文件夹。
|
||||
|
||||
说明：
|
||||
- dzkbw.com 本身是导航站，教材正文托管在国家中小学智慧教育平台（basic.smartedu.cn）。
|
||||
- 本脚本会解析出每个章节的官方直达链接，写入各本书的 Markdown 索引。
|
||||
- 可开启 --search，使用 DuckDuckGo Lite 按章节标题搜索网络补充资料。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
BASE_URL = "http://www.dzkbw.com"
|
||||
SMARTEDU_BASE = "https://basic.smartedu.cn/tchMaterial/detail"
|
||||
DDG_URL = "https://lite.duckduckgo.com/lite/"
|
||||
JINA_BASE = "https://r.jina.ai/http://"
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
PREFERRED_VERSIONS = [
|
||||
"人教版", "部编版", "北师大版", "苏教版",
|
||||
"外研版", "译林版", "教科版", "青岛版", "北京版",
|
||||
"沪教版", "冀教版", "鲁教版", "湘教版", "浙教版",
|
||||
]
|
||||
|
||||
GRADE_RE = re.compile(r"(五四制)?\s*([一二三四五六七八九十]+年级|高[一二三]|初[一二三])([上下]?)册?")
|
||||
SAFE_RE = re.compile(r'[\\/*?:"<>|]+')
|
||||
|
||||
KINDERGARTEN_STAGES = ["小班", "中班", "大班"]
|
||||
KINDERGARTEN_SUBJECTS = ["语言", "健康", "社会", "科学", "艺术", "数学启蒙", "英语启蒙"]
|
||||
|
||||
|
||||
class Cache:
|
||||
def __init__(self, path: Path):
|
||||
self.path = Path(path)
|
||||
self.data = {}
|
||||
if self.path.exists():
|
||||
try:
|
||||
self.data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
self.data = {}
|
||||
|
||||
def get(self, url: str):
|
||||
return self.data.get(url)
|
||||
|
||||
def set(self, url: str, value: str):
|
||||
self.data[url] = value
|
||||
self.save()
|
||||
|
||||
def save(self):
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(self.data, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def get_session():
|
||||
s = requests.Session()
|
||||
s.headers.update(HEADERS)
|
||||
retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
|
||||
s.mount("http://", HTTPAdapter(max_retries=retries))
|
||||
s.mount("https://", HTTPAdapter(max_retries=retries))
|
||||
return s
|
||||
|
||||
|
||||
def fetch_text(url: str, session: requests.Session, cache: Cache, delay: float = 0.3) -> str:
|
||||
cached = cache.get(url)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
resp = session.get(url, timeout=20)
|
||||
resp.raise_for_status()
|
||||
if "dzkbw.com" in url:
|
||||
text = resp.content.decode("gbk", errors="ignore")
|
||||
else:
|
||||
text = resp.text
|
||||
cache.set(url, text)
|
||||
return text
|
||||
|
||||
|
||||
def dzkbw_decode(encoded: str) -> str:
|
||||
n = len(encoded)
|
||||
k = 3 + n % 4
|
||||
acc = n * 7
|
||||
half = n // 2
|
||||
encoded = encoded[n - half:] + encoded[:n - half]
|
||||
out = []
|
||||
for i in range(1, n + 1):
|
||||
ch = encoded[i - 1]
|
||||
code = ord(ch) + acc % k + k // 2
|
||||
out.append(chr(code))
|
||||
acc = ord(out[-1])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def extract_content_id(chapter_html: str) -> str | None:
|
||||
m = re.search(r'<a\s+id=[\'"]gourl[\'"]\s+[^>]*?href="(/go/[^"]+)"', chapter_html, re.I)
|
||||
if not m:
|
||||
return None
|
||||
# 必须直接解析原始 href，否则 url 参数里的 %23 会被提前解码为 #，导致查询字符串被截断。
|
||||
href = m.group(1)
|
||||
q = urllib.parse.parse_qs(urllib.parse.urlparse(href).query)
|
||||
encoded = q.get("url", [""])[0]
|
||||
if not encoded:
|
||||
return None
|
||||
encoded = urllib.parse.unquote(encoded)
|
||||
decoded = dzkbw_decode(encoded)
|
||||
cm = re.search(r"contentId=([0-9a-f\-]+)", decoded)
|
||||
return cm.group(1) if cm else None
|
||||
|
||||
|
||||
def smartedu_url(content_id: str, page_code: str) -> str:
|
||||
page = page_code.zfill(3)
|
||||
return (
|
||||
f"{SMARTEDU_BASE}?contentType=assets_document"
|
||||
f"&contentId={content_id}&catalogType=tchMaterial"
|
||||
f"&subCatalog=tchMaterial&page={page}"
|
||||
)
|
||||
|
||||
|
||||
def safe_name(s: str) -> str:
|
||||
return SAFE_RE.sub("_", s).strip(" ._")
|
||||
|
||||
|
||||
def parse_home(html: str):
|
||||
"""返回 [(stage, stage_slug, subject_slug, subject_name), ...]"""
|
||||
# 支持按阶段找课本和年级找课本两个导航区
|
||||
patterns = [
|
||||
(r'按阶段找课本.*?</li>', True),
|
||||
(r'<li class="xl">按年级找课本.*?</li>', True),
|
||||
]
|
||||
seen = set()
|
||||
items = []
|
||||
for pat, use_in_list in patterns:
|
||||
m = re.search(pat, html, re.S | re.I)
|
||||
if not m:
|
||||
continue
|
||||
nav = m.group(0)
|
||||
for a in re.finditer(
|
||||
r'<a\s+href="/books/(xiaoxue|chuzhong|gaozhong)-([^"/]+)/"\s+title="([^"]+)"[^>]*>([^<]+)</a>',
|
||||
nav,
|
||||
re.I,
|
||||
):
|
||||
stage_slug, subject_slug, title_attr, text = a.groups()
|
||||
stage_map = {"xiaoxue": "小学", "chuzhong": "初中", "gaozhong": "高中"}
|
||||
stage = stage_map[stage_slug]
|
||||
subject_name = text.replace(stage, "").strip()
|
||||
key = (stage, subject_slug)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append((stage, stage_slug, subject_slug, subject_name))
|
||||
return items
|
||||
|
||||
|
||||
def parse_subject_page(html: str, stage: str, subject_name: str, only_preferred: bool = True):
|
||||
"""返回图书列表，按版本分组过滤。"""
|
||||
sections = re.split(r'<DIV\s+class=i_d_tc[^>]*>', html, flags=re.I)
|
||||
books = []
|
||||
for sec in sections[1:]:
|
||||
header_match = re.match(r"([^<]+)</DIV>", sec, re.I)
|
||||
if not header_match:
|
||||
continue
|
||||
version_heading = header_match.group(1).strip()
|
||||
if only_preferred and not any(v in version_heading for v in PREFERRED_VERSIONS):
|
||||
continue
|
||||
ul_part = sec.split("</UL>", 1)[0]
|
||||
for li in re.finditer(r"<LI[^>]*>(.*?)</LI>", ul_part, re.S | re.I):
|
||||
li_html = li.group(1)
|
||||
m1 = re.search(r'<A\s+class="mba"\s+href="([^"]+)"', li_html, re.I)
|
||||
m2 = re.search(
|
||||
r'<A\s+class="ih3"\s+href="[^"]+"\s+title="([^"]+)"[^>]*>([^<]*)</A>',
|
||||
li_html,
|
||||
re.I,
|
||||
)
|
||||
if not (m1 and m2):
|
||||
continue
|
||||
book_url = m1.group(1)
|
||||
book_title = m2.group(1).strip()
|
||||
gm = GRADE_RE.search(book_title)
|
||||
grade = gm.group(2) if gm else "其他"
|
||||
books.append(
|
||||
{
|
||||
"version_heading": version_heading,
|
||||
"url": book_url,
|
||||
"title": book_title,
|
||||
"grade": grade,
|
||||
}
|
||||
)
|
||||
return books
|
||||
|
||||
|
||||
def parse_book_page(html: str, book_url: str):
|
||||
title_match = re.search(r"<title>([^<]+)</title>", html, re.I)
|
||||
book_title = title_match.group(1).strip() if title_match else ""
|
||||
|
||||
book_path = book_url.rstrip("/")
|
||||
|
||||
# 封面图：优先匹配本书路径下的 lazy 图片
|
||||
cover_match = re.search(
|
||||
rf'<img[^>]*?class=[\'"]lazy[\'"][^>]*?data-original="({re.escape(book_path)}/cover\.jpg)"[^>]*?>',
|
||||
html,
|
||||
re.I,
|
||||
)
|
||||
cover = cover_match.group(1) if cover_match else ""
|
||||
|
||||
# 目录：本书目录页链接统一为 /books/.../{book_id}/{code}.htm
|
||||
toc = []
|
||||
seen = set()
|
||||
pattern = rf'<A\s+[^>]*?href="({re.escape(book_path)}/([0-9]*)\.htm)"[^>]*>(.*?)</A>'
|
||||
for a in re.finditer(pattern, html, re.S | re.I):
|
||||
href, code, txt = a.groups()
|
||||
code = (code or "000").zfill(3)
|
||||
if href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
title = re.sub(r"<[^>]+>", "", txt).strip()
|
||||
if title:
|
||||
toc.append(
|
||||
{
|
||||
"code": code,
|
||||
"title": title,
|
||||
"dzkbw_url": BASE_URL + href,
|
||||
}
|
||||
)
|
||||
return {"title": book_title, "cover": cover, "toc": toc}
|
||||
|
||||
|
||||
def search_ddg(query: str, max_results: int = 3):
|
||||
headers = {
|
||||
"User-Agent": HEADERS["User-Agent"],
|
||||
"Accept": "text/html",
|
||||
"Referer": "https://lite.duckduckgo.com/lite/",
|
||||
}
|
||||
try:
|
||||
resp = requests.post(
|
||||
DDG_URL, data={"q": query, "kl": "zh-CN"}, headers=headers, timeout=20
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
print(f" [search error] {e}")
|
||||
return []
|
||||
results = []
|
||||
for m in re.finditer(
|
||||
r'<a\s+rel="nofollow"\s+href="([^"]+)"\s+class=[\'"]result-link[\'"]\s*>([^<]+)</a>',
|
||||
resp.text,
|
||||
re.I,
|
||||
):
|
||||
url = m.group(1)
|
||||
title = re.sub(r"<[^>]+>", "", m.group(2)).strip()
|
||||
if not title or not url.startswith("http"):
|
||||
continue
|
||||
results.append({"title": title, "url": url})
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def write_book_markdown(
|
||||
output_root: Path,
|
||||
stage: str,
|
||||
grade: str,
|
||||
subject_name: str,
|
||||
book: dict,
|
||||
info: dict,
|
||||
supplements: dict,
|
||||
):
|
||||
folder = output_root / safe_name(stage) / safe_name(grade) / safe_name(subject_name)
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
version = safe_name(book["version_heading"].replace(stage, "").replace(subject_name, "").strip() or "未知版本")
|
||||
filename = f"{version}_{safe_name(book['title'])}.md"
|
||||
filepath = folder / filename
|
||||
|
||||
lines = []
|
||||
lines.append(f"# {book['title']}")
|
||||
lines.append("")
|
||||
lines.append(f"- **学段**：{stage}")
|
||||
lines.append(f"- **年级**：{book['grade']}")
|
||||
lines.append(f"- **科目**：{subject_name}")
|
||||
if book.get("version_heading"):
|
||||
lines.append(f"- **版本**：{book['version_heading']}")
|
||||
if book.get("url"):
|
||||
lines.append(f"- **电子课本网目录**：{BASE_URL}{book['url']}")
|
||||
else:
|
||||
lines.append(f"- **资料来源**：网络搜索整理")
|
||||
if info.get("cover"):
|
||||
lines.append(f"- **封面图**：{BASE_URL}{info['cover']}")
|
||||
lines.append("")
|
||||
lines.append("## 目录与官方阅读链接")
|
||||
lines.append("")
|
||||
lines.append("| 序号 | 章节 | 电子课本网 | 智慧教育平台 |")
|
||||
lines.append("|------|------|------------|--------------|")
|
||||
for ch in info["toc"]:
|
||||
smart = ch.get("smartedu_url", "")
|
||||
dzkbw = ch.get("dzkbw_url", "")
|
||||
dzkbw_link = f"[查看]({dzkbw})" if dzkbw else "-"
|
||||
smart_link = f"[官方阅读]({smart})" if smart else "-"
|
||||
lines.append(
|
||||
f"| {ch['code']} | {ch['title']} | {dzkbw_link} | {smart_link} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if supplements:
|
||||
lines.append("## 网络补充资料")
|
||||
lines.append("")
|
||||
for ch_title, results in supplements.items():
|
||||
lines.append(f"### {ch_title}")
|
||||
if results:
|
||||
for r in results:
|
||||
lines.append(f"- [{r['title']}]({r['url']})")
|
||||
else:
|
||||
lines.append("- 未找到结果")
|
||||
lines.append("")
|
||||
|
||||
filepath.write_text("\n".join(lines), encoding="utf-8")
|
||||
return filepath
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="构建义务教育阶段教材目录语料")
|
||||
parser.add_argument("--output", default="corpus/义务教育教材", help="输出根目录")
|
||||
parser.add_argument("--cache", default=".cache/dzkbw_cache.json", help="HTTP 缓存文件")
|
||||
parser.add_argument("--delay", type=float, default=0.25, help="请求间隔（秒）")
|
||||
parser.add_argument("--max-books-per-subject", type=int, default=0, help="每科目最多处理的书本数，0 为不限")
|
||||
parser.add_argument("--only-preferred", action="store_true", default=True, help="仅保留主流版本")
|
||||
parser.add_argument("--search", action="store_true", help="是否按章节搜索网络补充资料")
|
||||
parser.add_argument("--search-chapters", type=int, default=5, help="每本书搜索网络补充资料的章节数")
|
||||
parser.add_argument("--search-results", type=int, default=3, help="每章节保留的搜索结果数")
|
||||
args = parser.parse_args()
|
||||
|
||||
output_root = Path(args.output).resolve()
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
cache = Cache(Path(args.cache).resolve())
|
||||
session = get_session()
|
||||
|
||||
print(f"输出目录：{output_root}")
|
||||
print("正在解析 dzkbw.com 首页...")
|
||||
home_html = fetch_text(BASE_URL + "/", session, cache, delay=args.delay)
|
||||
subjects = parse_home(home_html)
|
||||
print(f"发现义务教育阶段科目：{len(subjects)} 个")
|
||||
|
||||
total_books = 0
|
||||
for stage, stage_slug, subject_slug, subject_name in subjects:
|
||||
subject_url = f"{BASE_URL}/books/{stage_slug}-{subject_slug}/"
|
||||
print(f"\n[{stage}] {subject_name} -> {subject_url}")
|
||||
try:
|
||||
subj_html = fetch_text(subject_url, session, cache, delay=args.delay)
|
||||
except Exception as e:
|
||||
print(f" 跳过（获取失败）：{e}")
|
||||
continue
|
||||
|
||||
books = parse_subject_page(subj_html, stage, subject_name, args.only_preferred)
|
||||
if not books:
|
||||
print(" 没有匹配到书本")
|
||||
continue
|
||||
if args.max_books_per_subject:
|
||||
books = books[: args.max_books_per_subject]
|
||||
|
||||
for book in books:
|
||||
print(f" - {book['title']} ({book['version_heading']})")
|
||||
try:
|
||||
book_html = fetch_text(
|
||||
BASE_URL + book["url"], session, cache, delay=args.delay
|
||||
)
|
||||
info = parse_book_page(book_html, book["url"])
|
||||
if not info["toc"]:
|
||||
print(" 未解析到目录，跳过")
|
||||
continue
|
||||
|
||||
first_chapter_html = fetch_text(
|
||||
info["toc"][0]["dzkbw_url"], session, cache, delay=args.delay
|
||||
)
|
||||
content_id = extract_content_id(first_chapter_html)
|
||||
if content_id:
|
||||
for ch in info["toc"]:
|
||||
ch["smartedu_url"] = smartedu_url(content_id, ch["code"])
|
||||
|
||||
supplements = {}
|
||||
if args.search:
|
||||
for ch in info["toc"][: args.search_chapters]:
|
||||
q = f"{book['title']} {ch['title']}"
|
||||
supplements[ch["title"]] = search_ddg(
|
||||
q, max_results=args.search_results
|
||||
)
|
||||
if args.delay > 0:
|
||||
time.sleep(args.delay)
|
||||
|
||||
filepath = write_book_markdown(
|
||||
output_root, stage, book["grade"], subject_name, book, info, supplements
|
||||
)
|
||||
total_books += 1
|
||||
print(f" 已生成：{filepath.relative_to(output_root)}")
|
||||
except Exception as e:
|
||||
print(f" 处理失败：{e}")
|
||||
continue
|
||||
|
||||
print(f"\n完成，共生成 {total_books} 本书的索引。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
128
scripts/fetch_corpus.py
Normal file
128
scripts/fetch_corpus.py
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从维基百科 API 抓取九年义务教育全科目中文语料。
|
||||
|
||||
科目覆盖：语文(古诗词/文言文) 数学 英语 化学 政治 历史 地理 物理 生物
|
||||
每个科目抓多个核心词条的纯文本摘要，落地到 corpus/<subject>.txt
|
||||
"""
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
OUT_DIR = Path(__file__).parent.parent / 'corpus'
|
||||
OUT_DIR.mkdir(exist_ok=True)
|
||||
|
||||
# 各科目核心词条（九年义务教育范围）
|
||||
SUBJECTS = {
|
||||
'语文': [
|
||||
'唐诗', '宋词', '元曲', '文言文', '李白', '杜甫', '白居易',
|
||||
'苏轼', '李清照', '辛弃疾', '鲁迅', '朱自清', '老舍',
|
||||
'三字经', '弟子规', '千字文', '百家姓', '论语', '孟子',
|
||||
],
|
||||
'数学': [
|
||||
'加减乘除', '分数', '小数', '百分比', '方程', '几何',
|
||||
'三角形', '圆', '正方形', '长方形', '平行四边形',
|
||||
'面积', '体积', '整数', '有理数', '无理数',
|
||||
'代数', '函数', '勾股定理', '圆周率',
|
||||
],
|
||||
'英语': [
|
||||
'英文字母', '英语语法', '英语动词', '英语名词',
|
||||
'英语形容词', '英语时态', '现在进行时', '一般现在时',
|
||||
'一般过去时', '一般将来时', '英语单词',
|
||||
],
|
||||
'化学': [
|
||||
'化学元素', '原子', '分子', '离子', '化合物', '混合物',
|
||||
'氧气', '氢气', '二氧化碳', '水', '酸', '碱', '盐',
|
||||
'化学反应', '氧化还原反应', '燃烧', '溶液', '金属',
|
||||
'元素周期表', '碳', '铁', '铜', '铝',
|
||||
],
|
||||
'政治': [
|
||||
'公民', '权利', '义务', '法律', '宪法', '人民代表大会制度',
|
||||
'中国共产党', '社会主义', '改革开放', '社会主义核心价值观',
|
||||
'国家安全', '消费者权益', '未成年人保护法',
|
||||
],
|
||||
'历史': [
|
||||
'中国历史', '夏朝', '商朝', '周朝', '秦朝', '汉朝',
|
||||
'唐朝', '宋朝', '元朝', '明朝', '清朝',
|
||||
'鸦片战争', '辛亥革命', '中华人民共和国', '抗日战争',
|
||||
'丝绸之路', '四大发明', '孔子', '秦始皇', '汉武帝',
|
||||
],
|
||||
'地理': [
|
||||
'地球', '大气层', '水循环', '气候', '地形', '平原',
|
||||
'高原', '山地', '盆地', '丘陵', '河流', '湖泊',
|
||||
'海洋', '中国地理', '长江', '黄河', '珠江', '青藏高原',
|
||||
'塔里木盆地', '华北平原', '中国省级行政区', '北京', '上海',
|
||||
],
|
||||
'物理': [
|
||||
'力', '运动', '速度', '加速度', '牛顿运动定律',
|
||||
'重力', '摩擦力', '弹力', '浮力', '压强',
|
||||
'功', '能量', '动能', '势能', '机械能守恒',
|
||||
'电', '电流', '电压', '电阻', '欧姆定律',
|
||||
'磁', '电磁感应', '光', '反射', '折射', '声音',
|
||||
'热传导', '温度', '比热容',
|
||||
],
|
||||
'生物': [
|
||||
'细胞', '细胞膜', '细胞核', '细胞质', 'DNA',
|
||||
'光合作用', '呼吸作用', '新陈代谢',
|
||||
'植物', '动物', '细菌', '病毒', '真菌',
|
||||
'生态系统', '食物链', '生物多样性',
|
||||
'人体', '心脏', '肺', '肝脏', '胃',
|
||||
'血液循环', '神经系统', '消化系统', '生殖', '遗传',
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def fetch_extract(title: str) -> str:
|
||||
"""从维基百科 API 获取词条纯文本摘要"""
|
||||
api = "https://zh.wikipedia.org/w/api.php"
|
||||
params = urllib.parse.urlencode({
|
||||
'action': 'query',
|
||||
'titles': title,
|
||||
'prop': 'extracts',
|
||||
'explaintext': '1',
|
||||
'exsectionformat': 'plain',
|
||||
'format': 'json',
|
||||
'redirects': '1',
|
||||
})
|
||||
url = f"{api}?{params}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'JspaceAI/1.0 (educational corpus)'})
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode('utf-8'))
|
||||
pages = data.get('query', {}).get('pages', {})
|
||||
for pid, page in pages.items():
|
||||
if pid != '-1' and 'extract' in page:
|
||||
return page['extract']
|
||||
except Exception as e:
|
||||
print(f" [warn] {title}: {e}")
|
||||
return ''
|
||||
|
||||
|
||||
def main():
|
||||
total_chars = 0
|
||||
for subject, titles in SUBJECTS.items():
|
||||
out_file = OUT_DIR / f"{subject}.txt"
|
||||
texts = []
|
||||
for title in titles:
|
||||
print(f" 抓取 [{subject}] {title}...", end=' ', flush=True)
|
||||
text = fetch_extract(title)
|
||||
if text:
|
||||
# 截取前 2000 字符避免过长
|
||||
text = text[:2000]
|
||||
texts.append(f"## {title}\n{text}")
|
||||
print(f"{len(text)} 字符")
|
||||
else:
|
||||
print("无")
|
||||
time.sleep(2.0) # 礼貌延时，避免 429
|
||||
|
||||
content = f"\n\n=== {subject} ===\n\n" + "\n\n".join(texts)
|
||||
out_file.write_text(content, encoding='utf-8')
|
||||
print(f"[{subject}] 保存到 {out_file.name}: {len(content)} 字符")
|
||||
total_chars += len(content)
|
||||
|
||||
print(f"\n完成，共 {total_chars} 字符，{len(SUBJECTS)} 个科目文件")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
306
scripts/fetch_corpus_incremental.py
Normal file
306
scripts/fetch_corpus_incremental.py
Normal file
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
"""增量抓取九年义务教育全科目中文语料（维基百科 API）。
|
||||
|
||||
特性：
|
||||
- 增量：已有词条跳过，只补缺失的
|
||||
- 429 自动重试：等待 10 秒后重试，最多 5 次
|
||||
- 长延时（3秒）防限流
|
||||
- 每词条截取前 3000 字符
|
||||
- 断点续传：Ctrl+C 中断后再次运行会跳过已抓的
|
||||
|
||||
用法：
|
||||
python scripts/fetch_corpus_incremental.py # 抓全部科目
|
||||
python scripts/fetch_corpus_incremental.py 语文 数学 # 只抓指定科目
|
||||
"""
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import time
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
CORPUS_DIR = Path(__file__).parent.parent / 'corpus'
|
||||
CORPUS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 九年义务教育各科目核心词条（覆盖小学到初中知识点）
|
||||
SUBJECTS = {
|
||||
'语文': [
|
||||
# 古诗词与文言文（部编版教材重点篇目）
|
||||
'静夜思', '春晓', '登鹳雀楼', '望庐山瀑布', '绝句', '春望',
|
||||
'江雪', '游子吟', '清明', '枫桥夜泊', '寻隐者不遇',
|
||||
'咏鹅', '回乡偶书', '咏柳', '凉州词', '出塞',
|
||||
'芙蓉楼送辛渐', '送元二使安西', '九月九日忆山东兄弟',
|
||||
'赠汪伦', '黄鹤楼送孟浩然之广陵', '望天门山',
|
||||
'别董大', '早发白帝城', '赠花卿', '江南逢李龟年',
|
||||
'春夜喜雨', '江畔独步寻花', '渔歌子', '塞下曲',
|
||||
'望洞庭', '浪淘沙', '赋得古原草送别', '忆江南',
|
||||
'悯农', '蚕妇', '陶者', '梅花', '元日',
|
||||
'泊船瓜洲', '书湖阴先生壁', '六月二十七日望湖楼醉书',
|
||||
'饮湖上初晴后雨', '题西林壁', '惠崇春江晚景',
|
||||
'晓出净慈寺送林子方', '小池', '宿新市徐公店',
|
||||
'春日', '观书有感', '题临安邸', '游园不值',
|
||||
'乡村四月', '墨梅', '石灰吟', '竹石', '所见',
|
||||
'村居', '己亥杂诗',
|
||||
# 文言文
|
||||
'论语', '孟子', '大学', '中庸', '诗经',
|
||||
'世说新语', '聊斋志异', '西游记', '水浒传',
|
||||
'三国演义', '红楼梦',
|
||||
'岳阳楼记', '醉翁亭记', '爱莲说', '陋室铭',
|
||||
'桃花源记', '小石潭记', '记承天寺夜游',
|
||||
'送东阳马生序', '曹刿论战', '邹忌讽齐王纳谏',
|
||||
'出师表', '陈涉世家',
|
||||
# 作家
|
||||
'李白', '杜甫', '白居易', '王维', '苏轼',
|
||||
'李清照', '辛弃疾', '陆游', '鲁迅', '朱自清',
|
||||
'老舍', '巴金', '冰心',
|
||||
# 文学常识
|
||||
'唐诗', '宋词', '元曲', '文言文', '成语',
|
||||
],
|
||||
'数学': [
|
||||
'加法', '减法', '乘法', '除法', '四则运算',
|
||||
'分数', '小数', '百分比', '比例', '方程',
|
||||
'一元一次方程', '二元一次方程', '一元二次方程',
|
||||
'不等式', '一元一次不等式',
|
||||
'几何学', '三角形', '圆', '正方形', '长方形',
|
||||
'平行四边形', '梯形', '多边形', '正方体', '长方体',
|
||||
'圆柱', '圆锥', '球', '扇形',
|
||||
'面积', '体积', '周长', '角度', '弧',
|
||||
'整数', '有理数', '无理数', '实数', '复数',
|
||||
'代数', '函数', '一次函数', '二次函数', '反比例函数',
|
||||
'勾股定理', '圆周率', '素数', '因数', '倍数',
|
||||
'最大公约数', '最小公倍数', '绝对值', '平方根',
|
||||
'统计', '概率', '平均数', '中位数', '众数',
|
||||
'负数', '相反数', '倒数', '科学记数法',
|
||||
'相似三角形', '全等三角形', '平行线',
|
||||
'轴对称', '中心对称', '平移', '旋转',
|
||||
'坐标', '直角坐标系', '象限',
|
||||
],
|
||||
'英语': [
|
||||
'英文字母', '英语', '英语语法', '英语动词',
|
||||
'英语名词', '英语形容词', '英语副词', '英语代词',
|
||||
'英语介词', '英语连词', '英语冠词',
|
||||
'英语时态', '一般现在时', '现在进行时',
|
||||
'一般过去时', '过去进行时', '现在完成时',
|
||||
'一般将来时', '过去完成时', '过去完成进行时',
|
||||
'英语被动语态', '英语从句', '定语从句',
|
||||
'名词性从句', '状语从句', '宾语从句',
|
||||
'主语从句', '表语从句', '同位语从句',
|
||||
'英语虚拟语气', '英语不定式', '英语动名词', '英语分词',
|
||||
'英语单词', '英语发音', '英语拼写', '英语音标',
|
||||
],
|
||||
'化学': [
|
||||
'化学', '化学元素', '元素周期表', '原子', '分子',
|
||||
'离子', '化合物', '混合物', '纯净物', '单质',
|
||||
'氧气', '氢气', '氮气', '二氧化碳', '一氧化碳',
|
||||
'水', '盐酸', '硫酸', '硝酸', '氢氧化钠',
|
||||
'碳酸钙', '氯化钠', '碳酸氢钠', '氢氧化钙',
|
||||
'酸', '碱', '盐', '氧化物', '过氧化物',
|
||||
'化学反应', '化合反应', '分解反应', '置换反应',
|
||||
'氧化还原反应', '复分解反应', '燃烧', '爆炸',
|
||||
'溶液', '溶质', '溶剂', '溶解度', '浓度',
|
||||
'金属', '合金', '铁', '铜', '铝', '锌', '银',
|
||||
'碳', '金刚石', '石墨', '有机化合物', '甲烷',
|
||||
'乙醇', '乙酸', '蛋白质', '糖类', '脂肪',
|
||||
'化学式', '化学方程式', '摩尔', '相对原子质量',
|
||||
'原子结构', '电子', '质子', '中子', '同位素',
|
||||
'化学键', '离子键', '共价键', '金属键',
|
||||
],
|
||||
'政治': [
|
||||
'公民', '权利', '义务', '法律', '宪法',
|
||||
'刑法', '民法', '消费者权益', '未成年人保护',
|
||||
'人民代表大会制度', '全国人民代表大会',
|
||||
'中国共产党', '社会主义', '社会主义市场经济',
|
||||
'改革开放', '社会主义核心价值观',
|
||||
'国家安全', '爱国主义', '集体主义',
|
||||
'民主', '自由', '平等', '公正', '法治',
|
||||
'民族团结', '一国两制', '和平共处五项原则',
|
||||
'联合国', '世界贸易组织', '经济全球化',
|
||||
'人民代表大会', '政治协商制度', '民族区域自治',
|
||||
'基层群众自治', '依法治国', '以德治国',
|
||||
],
|
||||
'历史': [
|
||||
'中国历史', '夏朝', '商朝', '西周', '东周',
|
||||
'春秋时期', '战国时期', '秦朝', '西汉', '东汉',
|
||||
'三国', '西晋', '东晋', '南北朝', '隋朝', '唐朝',
|
||||
'宋朝', '辽朝', '金朝', '元朝', '明朝', '清朝',
|
||||
'中华民国', '中华人民共和国',
|
||||
'鸦片战争', '太平天国运动', '洋务运动', '甲午战争',
|
||||
'戊戌变法', '义和团运动', '辛亥革命',
|
||||
'新文化运动', '五四运动', '中国共产党成立',
|
||||
'北伐战争', '土地革命', '长征',
|
||||
'抗日战争', '解放战争', '新中国成立',
|
||||
'丝绸之路', '四大发明', '郑和下西洋',
|
||||
'孔子', '老子', '孟子', '庄子', '韩非子', '墨子',
|
||||
'秦始皇', '汉武帝', '唐太宗', '宋太祖',
|
||||
'成吉思汗', '朱元璋', '康熙帝',
|
||||
'世界历史', '古埃及', '古希腊', '古罗马',
|
||||
'文艺复兴', '工业革命', '第一次世界大战',
|
||||
'第二次世界大战', '冷战', '欧洲联盟',
|
||||
],
|
||||
'地理': [
|
||||
'地球', '地球仪', '经线', '纬线', '赤道',
|
||||
'本初子午线', '时区', '国际日期变更线',
|
||||
'大气层', '对流层', '水循环', '气候',
|
||||
'热带', '温带', '寒带', '季风', '降水',
|
||||
'地形', '平原', '高原', '山地', '盆地', '丘陵',
|
||||
'河流', '湖泊', '海洋', '海峡', '岛屿', '半岛',
|
||||
'中国地理', '中国行政区划', '省级行政区',
|
||||
'长江', '黄河', '珠江', '淮河', '黑龙江',
|
||||
'青藏高原', '内蒙古高原', '黄土高原', '云贵高原',
|
||||
'塔里木盆地', '准噶尔盆地', '柴达木盆地', '四川盆地',
|
||||
'华北平原', '东北平原', '长江中下游平原',
|
||||
'喜马拉雅山脉', '昆仑山脉', '秦岭',
|
||||
'北京', '上海', '广州', '深圳', '天津', '重庆',
|
||||
'香港', '澳门', '台湾',
|
||||
'世界地理', '亚洲', '欧洲', '非洲',
|
||||
'北美洲', '南美洲', '大洋洲', '南极洲',
|
||||
'太平洋', '大西洋', '印度洋', '北冰洋',
|
||||
'撒哈拉沙漠', '亚马逊雨林', '尼罗河',
|
||||
'人口', '城市', '农业', '工业', '交通运输',
|
||||
],
|
||||
'物理': [
|
||||
'物理学', '力', '运动', '速度', '加速度',
|
||||
'牛顿运动定律', '牛顿第一定律', '牛顿第二定律',
|
||||
'牛顿第三定律', '惯性', '质量', '密度',
|
||||
'重力', '摩擦力', '弹力', '浮力', '压强',
|
||||
'大气压强', '液体压强', '液压千斤顶',
|
||||
'功', '功率', '能量', '动能', '势能',
|
||||
'机械能', '机械能守恒', '能量守恒定律',
|
||||
'杠杆', '滑轮', '斜面', '轮轴',
|
||||
'电', '电荷', '电流', '电压', '电阻',
|
||||
'欧姆定律', '串联', '并联', '电功率', '电能',
|
||||
'磁', '磁场', '磁体', '电磁感应', '电磁铁',
|
||||
'电动机', '发电机', '变压器',
|
||||
'光', '光的反射', '光的折射', '透镜',
|
||||
'凸透镜', '凹透镜', '光的色散', '颜色', '光谱',
|
||||
'声音', '声波', '音调', '响度', '音色',
|
||||
'回声', '超声波', '次声波', '噪声',
|
||||
'热', '温度', '温度计', '熔化', '凝固',
|
||||
'汽化', '液化', '升华', '凝华',
|
||||
'比热容', '热传导', '热对流', '热辐射',
|
||||
'物态变化', '沸腾', '蒸发', '内能',
|
||||
'比热', '热量', '热机', '热机效率',
|
||||
],
|
||||
'生物': [
|
||||
'生物学', '细胞', '细胞膜', '细胞核', '细胞质',
|
||||
'细胞壁', '叶绿体', '线粒体', '染色体', 'DNA',
|
||||
'基因', '遗传', '变异', '遗传学',
|
||||
'光合作用', '呼吸作用', '新陈代谢',
|
||||
'植物', '动物', '细菌', '病毒', '真菌',
|
||||
'苔藓植物', '蕨类植物', '种子植物', '裸子植物', '被子植物',
|
||||
'脊椎动物', '无脊椎动物', '哺乳动物', '鸟类',
|
||||
'爬行动物', '两栖动物', '鱼类', '昆虫',
|
||||
'生态系统', '食物链', '食物网', '生物多样性',
|
||||
'生产者', '消费者', '分解者', '生物圈',
|
||||
'人体', '心脏', '血管', '血液', '血液循环',
|
||||
'肺', '呼吸', '肝脏', '胃', '肠',
|
||||
'神经系统', '大脑', '脊髓', '神经元',
|
||||
'消化系统', '吸收', '排泄', '泌尿系统',
|
||||
'生殖', '发育', '青春期', '激素',
|
||||
'群落', '种群', '生态系统稳定性',
|
||||
'进化', '自然选择', '达尔文', '物种起源',
|
||||
'组织', '器官', '系统', '细胞分裂', '细胞分化',
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def fetch_extract(title: str, retries: int = 5) -> str:
|
||||
"""从维基百科 API 获取词条纯文本摘要，429 自动重试"""
|
||||
api = "https://zh.wikipedia.org/w/api.php"
|
||||
params = urllib.parse.urlencode({
|
||||
'action': 'query',
|
||||
'titles': title,
|
||||
'prop': 'extracts',
|
||||
'explaintext': '1',
|
||||
'exsectionformat': 'plain',
|
||||
'format': 'json',
|
||||
'redirects': '1',
|
||||
})
|
||||
url = f"{api}?{params}"
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
'User-Agent': 'JspaceAI/1.0 (educational corpus fetcher; contact@example.com)'
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
data = json.loads(resp.read().decode('utf-8'))
|
||||
pages = data.get('query', {}).get('pages', {})
|
||||
for pid, page in pages.items():
|
||||
if pid != '-1' and 'extract' in page:
|
||||
return page['extract'][:3000] # 截取前 3000 字符
|
||||
return '' # 词条不存在
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 10 * (attempt + 1)
|
||||
print(f" 429 限流，等待 {wait}s 后重试 ({attempt+1}/{retries})")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
print(f" HTTP {e.code}: {title}")
|
||||
return ''
|
||||
except Exception as e:
|
||||
print(f" 错误: {e}")
|
||||
time.sleep(5)
|
||||
return ''
|
||||
|
||||
|
||||
def get_existing_titles(filepath: Path) -> set:
|
||||
"""从已有文件读取已抓取的词条名"""
|
||||
if not filepath.exists():
|
||||
return set()
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
return set(re.findall(r'^## (.+)$', content, re.M))
|
||||
|
||||
|
||||
def main():
|
||||
subjects = sys.argv[1:] if len(sys.argv) > 1 else list(SUBJECTS.keys())
|
||||
|
||||
total_new = 0
|
||||
total_skip = 0
|
||||
|
||||
for subject in subjects:
|
||||
if subject not in SUBJECTS:
|
||||
print(f"未知科目: {subject}")
|
||||
continue
|
||||
|
||||
titles = SUBJECTS[subject]
|
||||
out_file = CORPUS_DIR / f"{subject}.txt"
|
||||
existing = get_existing_titles(out_file)
|
||||
|
||||
new_titles = [t for t in titles if t not in existing]
|
||||
skip_count = len(existing)
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[{subject}] 共 {len(titles)} 词条，已有 {skip_count}，新增 {len(new_titles)}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# 读取已有内容（追加模式）
|
||||
if out_file.exists():
|
||||
all_content = out_file.read_text(encoding='utf-8')
|
||||
else:
|
||||
all_content = f"=== {subject} ===\n"
|
||||
|
||||
for i, title in enumerate(new_titles):
|
||||
print(f" [{i+1}/{len(new_titles)}] {title}...", end=' ', flush=True)
|
||||
text = fetch_extract(title)
|
||||
if text:
|
||||
all_content += f"\n\n## {title}\n{text}\n"
|
||||
print(f"{len(text)} 字符")
|
||||
total_new += 1
|
||||
else:
|
||||
print("无")
|
||||
# 增量保存（每抓 5 条保存一次，防中断丢失）
|
||||
if (i + 1) % 5 == 0:
|
||||
out_file.write_text(all_content, encoding='utf-8')
|
||||
time.sleep(3) # 长延时防限流
|
||||
|
||||
out_file.write_text(all_content, encoding='utf-8')
|
||||
print(f"[{subject}] 完成，已保存到 {out_file.name}")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全部完成：新增 {total_new} 词条，跳过 {total_skip} 已有")
|
||||
print(f"语料目录: {CORPUS_DIR}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user