227 lines
11 KiB
Python
227 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""内容校验 + 目录生成（只用标准库，Python 3.10+）。
|
||
|
||
用法：
|
||
python3 tools/build_catalog.py # 校验并写出 catalog.json
|
||
python3 tools/build_catalog.py --check # 只校验，不写文件（CI / 提交前用）
|
||
python3 tools/build_catalog.py --strict # 额外要求：每个技术栈至少 1 课、1 个场景、1 份面试题
|
||
|
||
校验内容：frontmatter 字段和取值、id 唯一、prerequisites / related 指向存在的 id、
|
||
相对链接指向存在的文件、代码块围栏闭合、各类型必需的小节标题。规范见 CONTRIBUTING.md。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
CONTENT_DIRS = ["plan", "topics", "scenarios", "research"]
|
||
SCHEMA_VERSION = 1
|
||
|
||
TYPES = {"index", "lesson", "scenario", "interview", "project", "week", "research"}
|
||
DOMAINS = {"backend", "devops", "python", "ai", "frontend", "cs", "cross", "plan"}
|
||
LEVELS = {"basic", "intermediate", "advanced"}
|
||
QUESTION_LEVELS = ["基础", "进阶", "场景题"]
|
||
ID_RE = re.compile(r"^[a-z0-9]+(?:\.[a-z0-9-]+)+$")
|
||
REQUIRED = {"id", "title", "type", "domain"}
|
||
LIST_FIELDS = {"stack", "tags", "prerequisites", "related", "week"}
|
||
REQUIRED_SECTIONS = {
|
||
"lesson": ["解释说明", "案例", "代码示例", "易错点"],
|
||
"scenario": ["业务背景", "约束", "方案设计", "关键代码", "取舍", "易错点", "面试怎么讲"],
|
||
}
|
||
FENCE_RE = re.compile(r"^\s*(```|~~~)")
|
||
LINK_RE = re.compile(r"(?<!!)\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
|
||
QUESTION_RE = re.compile(r"^###\s+【(基础|进阶|场景题)】")
|
||
|
||
|
||
def parse_scalar(v: str):
|
||
v = v.strip()
|
||
if v == "":
|
||
return ""
|
||
if (v[0] == v[-1]) and v[0] in "\"'" and len(v) >= 2:
|
||
return v[1:-1]
|
||
if re.fullmatch(r"-?\d+", v):
|
||
return int(v)
|
||
if re.fullmatch(r"-?\d+\.\d+", v):
|
||
return float(v)
|
||
if v in ("true", "false"):
|
||
return v == "true"
|
||
return v
|
||
|
||
|
||
def parse_frontmatter(text: str) -> tuple[dict | None, str, list[str]]:
|
||
"""只支持规范里约定的子集：`键: 标量` 和 `键: [a, b]`（单行流式列表）。"""
|
||
errs: list[str] = []
|
||
if not text.startswith("---\n"):
|
||
return None, text, ["缺少 frontmatter（文件第一行必须是 ---）"]
|
||
end = text.find("\n---\n", 4)
|
||
if end < 0:
|
||
return None, text, ["frontmatter 没有结束的 ---"]
|
||
meta: dict = {}
|
||
for n, line in enumerate(text[4:end].splitlines(), 2):
|
||
if not line.strip() or line.lstrip().startswith("#"):
|
||
continue
|
||
m = re.match(r"^([a-z_]+):\s*(.*)$", line)
|
||
if not m:
|
||
errs.append(f"frontmatter 第 {n} 行无法解析：{line!r}")
|
||
continue
|
||
k, v = m.group(1), m.group(2).strip()
|
||
if v.startswith("["):
|
||
if not v.endswith("]"):
|
||
errs.append(f"frontmatter 字段 {k} 的列表必须写在一行内")
|
||
continue
|
||
inner = v[1:-1].strip()
|
||
meta[k] = [parse_scalar(x) for x in inner.split(",")] if inner else []
|
||
else:
|
||
meta[k] = parse_scalar(v)
|
||
return meta, text[end + 5:], errs
|
||
|
||
|
||
def scan_body(body: str) -> tuple[bool, list[str], list[str], dict[str, int], list[str]]:
|
||
"""返回：围栏是否闭合、二级标题、链接目标、按级别统计的面试题数、缺少参考答案或追问的题目。"""
|
||
in_fence, fence_mark = False, ""
|
||
headings, links = [], []
|
||
counts = {k: 0 for k in QUESTION_LEVELS}
|
||
questions: list[list] = [] # [标题, 有参考答案, 有追问]
|
||
for line in body.splitlines():
|
||
m = FENCE_RE.match(line)
|
||
if m:
|
||
if not in_fence:
|
||
in_fence, fence_mark = True, m.group(1)
|
||
elif m.group(1) == fence_mark and line.strip() == fence_mark:
|
||
in_fence = False
|
||
continue
|
||
if in_fence:
|
||
continue
|
||
if line.startswith("## "):
|
||
headings.append(line[3:].strip())
|
||
q = QUESTION_RE.match(line)
|
||
if q:
|
||
counts[q.group(1)] += 1
|
||
questions.append([line[4:].strip(), False, False])
|
||
elif line.startswith("#"):
|
||
if questions and line.startswith(("# ", "## ", "### ")):
|
||
questions.append([None, True, True]) # 题目之后的其他标题，结束上一题
|
||
elif questions:
|
||
questions[-1][1] |= line.startswith("**参考答案**")
|
||
questions[-1][2] |= line.startswith("**追问**")
|
||
links += LINK_RE.findall(line)
|
||
incomplete = [f"{t}（缺少{'“**参考答案**”' if not a else ''}{'“**追问**”' if not f else ''}）"
|
||
for t, a, f in questions if t and not (a and f)]
|
||
return not in_fence, headings, links, counts, incomplete
|
||
|
||
|
||
def main() -> int:
|
||
check_only = "--check" in sys.argv
|
||
strict = "--strict" in sys.argv
|
||
errors: list[str] = []
|
||
items: list[dict] = []
|
||
files = sorted(p for d in CONTENT_DIRS for p in (ROOT / d).rglob("*.md")
|
||
if "examples" not in p.relative_to(ROOT).parts and "node_modules" not in p.parts)
|
||
for path in files:
|
||
rel = path.relative_to(ROOT).as_posix()
|
||
text = path.read_text(encoding="utf-8")
|
||
meta, body, errs = parse_frontmatter(text)
|
||
errors += [f"{rel}: {e}" for e in errs]
|
||
if meta is None:
|
||
continue
|
||
for f in REQUIRED - meta.keys():
|
||
errors.append(f"{rel}: 缺少必填字段 {f}")
|
||
for f in LIST_FIELDS & meta.keys():
|
||
if not isinstance(meta[f], list):
|
||
errors.append(f"{rel}: 字段 {f} 必须是列表，例如 {f}: [a, b]")
|
||
t = meta.get("type")
|
||
if t not in TYPES:
|
||
errors.append(f"{rel}: type={t!r} 不在 {sorted(TYPES)} 中")
|
||
if meta.get("domain") not in DOMAINS:
|
||
errors.append(f"{rel}: domain={meta.get('domain')!r} 不在 {sorted(DOMAINS)} 中")
|
||
if not ID_RE.match(str(meta.get("id", ""))):
|
||
errors.append(f"{rel}: id={meta.get('id')!r} 格式不对（小写字母数字，用 . 分段）")
|
||
if t in ("lesson", "scenario", "interview") and meta.get("level") not in LEVELS:
|
||
errors.append(f"{rel}: {t} 必须有 level（{sorted(LEVELS)}）")
|
||
if "est_hours" in meta and not isinstance(meta["est_hours"], (int, float)):
|
||
errors.append(f"{rel}: est_hours 必须是数字")
|
||
for w in meta.get("week", []) if isinstance(meta.get("week"), list) else []:
|
||
if not isinstance(w, int) or not 1 <= w <= 16:
|
||
errors.append(f"{rel}: week 取值 {w!r} 必须是 1~16 的整数")
|
||
closed, headings, links, counts, incomplete = scan_body(body)
|
||
if t == "interview":
|
||
errors += [f"{rel}: 面试题 {q}" for q in incomplete]
|
||
if not closed:
|
||
errors.append(f"{rel}: 代码块围栏没有闭合")
|
||
for sec in REQUIRED_SECTIONS.get(t, []):
|
||
if not any(sec in h for h in headings):
|
||
errors.append(f"{rel}: {t} 缺少小节“## {sec}”")
|
||
if t == "interview" and sum(counts.values()) == 0:
|
||
errors.append(f"{rel}: 面试题文件没有题目（标题格式：### 【基础】问题）")
|
||
for target in links:
|
||
if re.match(r"^[a-z]+:", target) or target.startswith("#"):
|
||
continue
|
||
dest = (path.parent / target.split("#", 1)[0]).resolve()
|
||
if not dest.exists():
|
||
errors.append(f"{rel}: 链接指向的文件不存在：{target}")
|
||
item = {
|
||
"id": meta.get("id"), "title": meta.get("title"), "type": t, "domain": meta.get("domain"),
|
||
"stack": meta.get("stack", []), "tags": meta.get("tags", []), "level": meta.get("level"),
|
||
"prerequisites": meta.get("prerequisites", []), "related": meta.get("related", []),
|
||
"est_hours": meta.get("est_hours"), "week": meta.get("week", []), "order": meta.get("order"),
|
||
"path": rel, "sections": headings,
|
||
}
|
||
if t == "interview":
|
||
item["questions"] = counts
|
||
items.append(item)
|
||
|
||
ids = {}
|
||
for it in items:
|
||
if it["id"] in ids:
|
||
errors.append(f"{it['path']}: id {it['id']} 与 {ids[it['id']]} 重复")
|
||
ids[it["id"]] = it["path"]
|
||
for it in items:
|
||
for f in ("prerequisites", "related"):
|
||
for ref in it[f] if isinstance(it[f], list) else []:
|
||
if ref not in ids:
|
||
errors.append(f"{it['path']}: {f} 引用了不存在的 id：{ref}")
|
||
|
||
stacks = []
|
||
for it in items:
|
||
if it["type"] == "index" and it["path"].startswith("topics/") and it["path"].count("/") == 3:
|
||
sid = it["stack"][0] if it["stack"] else None
|
||
mine = [x for x in items if x["path"].startswith(it["path"].rsplit("/", 1)[0] + "/")]
|
||
q = {k: 0 for k in QUESTION_LEVELS}
|
||
for x in mine:
|
||
for k, v in x.get("questions", {}).items():
|
||
q[k] += v
|
||
stack_dir = ROOT / it["path"].rsplit("/", 1)[0]
|
||
examples = sorted(p.relative_to(ROOT).as_posix() for p in (stack_dir / "examples").glob("*")) \
|
||
if (stack_dir / "examples").exists() else []
|
||
entry = {"stack": sid, "domain": it["domain"], "title": it["title"], "index": it["path"],
|
||
"lessons": sum(x["type"] == "lesson" for x in mine),
|
||
"scenarios": sum(x["type"] == "scenario" for x in mine),
|
||
"interview_questions": q, "examples": examples}
|
||
stacks.append(entry)
|
||
if strict and (entry["lessons"] == 0 or entry["scenarios"] == 0 or sum(q.values()) == 0):
|
||
errors.append(f"{it['path']}: 严格模式要求技术栈 {sid} 至少有 1 课、1 个场景和面试题")
|
||
stacks.sort(key=lambda s: (s["domain"], s["stack"] or ""))
|
||
|
||
if errors:
|
||
print(f"校验失败：{len(errors)} 个问题")
|
||
for e in errors:
|
||
print(" -", e)
|
||
return 1
|
||
q_total = {k: sum(s["interview_questions"][k] for s in stacks) for k in QUESTION_LEVELS}
|
||
print(f"校验通过：{len(items)} 个文件，{len(stacks)} 个技术栈，"
|
||
f"{sum(i['type'] == 'lesson' for i in items)} 课，{sum(i['type'] == 'scenario' for i in items)} 个场景，"
|
||
f"面试题 {sum(q_total.values())} 道（{'、'.join(f'{k} {v}' for k, v in q_total.items())}）")
|
||
if not check_only:
|
||
catalog = {"schema_version": SCHEMA_VERSION, "generated_by": "tools/build_catalog.py",
|
||
"stacks": stacks, "items": sorted(items, key=lambda x: x["id"])}
|
||
(ROOT / "catalog.json").write_text(json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print("已写出 catalog.json")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|