内容结构化改造（阶段 1）：新增 topics/scenarios 目录、内容规范、frontmatter schema、校验与目录生成脚本、31 个技术栈首页骨架，周文件加 frontmatter
This commit is contained in:
Shikui Wang committed
1 parent
32b111606f
commit
0bae9916a9
58 files changed
+2995
-5
No files matched your search
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env python3
|
||||
"""内容校验 + 目录生成（只用标准库，Python 3.10+）。
|
||||
|
||||
用法：
|
||||
python3 tools/build_catalog.py # 校验并写出 catalog.json
|
||||
python3 tools/build_catalog.py --check # 只校验，不写文件（CI / 提交前用）
|
||||
python3 tools/build_catalog.py --strict # 额外要求：每个技术栈至少 1 课、1 个场景、1 份面试题
|
||||
|
||||
校验内容：frontmatter 字段和取值、id 唯一、prerequisites / related 指向存在的 id、
|
||||
相对链接指向存在的文件、代码块围栏闭合、各类型必需的小节标题。规范见 CONTRIBUTING.md。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONTENT_DIRS = ["plan", "topics", "scenarios"]
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
TYPES = {"index", "lesson", "scenario", "interview", "project", "week"}
|
||||
DOMAINS = {"backend", "devops", "python", "ai", "frontend", "cross", "plan"}
|
||||
LEVELS = {"basic", "intermediate", "advanced"}
|
||||
QUESTION_LEVELS = ["基础", "进阶", "场景题"]
|
||||
ID_RE = re.compile(r"^[a-z0-9]+(?:\.[a-z0-9-]+)+$")
|
||||
REQUIRED = {"id", "title", "type", "domain"}
|
||||
LIST_FIELDS = {"stack", "tags", "prerequisites", "related", "week"}
|
||||
REQUIRED_SECTIONS = {
|
||||
"lesson": ["解释说明", "案例", "代码示例", "易错点"],
|
||||
"scenario": ["业务背景", "约束", "方案设计", "关键代码", "取舍", "易错点", "面试怎么讲"],
|
||||
}
|
||||
FENCE_RE = re.compile(r"^\s*(```|~~~)")
|
||||
LINK_RE = re.compile(r"(?<!!)\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
|
||||
QUESTION_RE = re.compile(r"^###\s+【(基础|进阶|场景题)】")
|
||||
|
||||
|
||||
def parse_scalar(v: str):
|
||||
v = v.strip()
|
||||
if v == "":
|
||||
return ""
|
||||
if (v[0] == v[-1]) and v[0] in "\"'" and len(v) >= 2:
|
||||
return v[1:-1]
|
||||
if re.fullmatch(r"-?\d+", v):
|
||||
return int(v)
|
||||
if re.fullmatch(r"-?\d+\.\d+", v):
|
||||
return float(v)
|
||||
if v in ("true", "false"):
|
||||
return v == "true"
|
||||
return v
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict | None, str, list[str]]:
|
||||
"""只支持规范里约定的子集：`键: 标量` 和 `键: [a, b]`（单行流式列表）。"""
|
||||
errs: list[str] = []
|
||||
if not text.startswith("---\n"):
|
||||
return None, text, ["缺少 frontmatter（文件第一行必须是 ---）"]
|
||||
end = text.find("\n---\n", 4)
|
||||
if end < 0:
|
||||
return None, text, ["frontmatter 没有结束的 ---"]
|
||||
meta: dict = {}
|
||||
for n, line in enumerate(text[4:end].splitlines(), 2):
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
m = re.match(r"^([a-z_]+):\s*(.*)$", line)
|
||||
if not m:
|
||||
errs.append(f"frontmatter 第 {n} 行无法解析：{line!r}")
|
||||
continue
|
||||
k, v = m.group(1), m.group(2).strip()
|
||||
if v.startswith("["):
|
||||
if not v.endswith("]"):
|
||||
errs.append(f"frontmatter 字段 {k} 的列表必须写在一行内")
|
||||
continue
|
||||
inner = v[1:-1].strip()
|
||||
meta[k] = [parse_scalar(x) for x in inner.split(",")] if inner else []
|
||||
else:
|
||||
meta[k] = parse_scalar(v)
|
||||
return meta, text[end + 5:], errs
|
||||
|
||||
|
||||
def scan_body(body: str) -> tuple[bool, list[str], list[str], dict[str, int]]:
|
||||
"""返回：围栏是否闭合、二级标题、链接目标、按级别统计的面试题数。"""
|
||||
in_fence, fence_mark = False, ""
|
||||
headings, links = [], []
|
||||
counts = {k: 0 for k in QUESTION_LEVELS}
|
||||
for line in body.splitlines():
|
||||
m = FENCE_RE.match(line)
|
||||
if m:
|
||||
if not in_fence:
|
||||
in_fence, fence_mark = True, m.group(1)
|
||||
elif m.group(1) == fence_mark and line.strip() == fence_mark:
|
||||
in_fence = False
|
||||
continue
|
||||
if in_fence:
|
||||
continue
|
||||
if line.startswith("## "):
|
||||
headings.append(line[3:].strip())
|
||||
q = QUESTION_RE.match(line)
|
||||
if q:
|
||||
counts[q.group(1)] += 1
|
||||
links += LINK_RE.findall(line)
|
||||
return not in_fence, headings, links, counts
|
||||
|
||||
|
||||
def main() -> int:
|
||||
check_only = "--check" in sys.argv
|
||||
strict = "--strict" in sys.argv
|
||||
errors: list[str] = []
|
||||
items: list[dict] = []
|
||||
files = sorted(p for d in CONTENT_DIRS for p in (ROOT / d).rglob("*.md")
|
||||
if "examples" not in p.relative_to(ROOT).parts and "node_modules" not in p.parts)
|
||||
for path in files:
|
||||
rel = path.relative_to(ROOT).as_posix()
|
||||
text = path.read_text(encoding="utf-8")
|
||||
meta, body, errs = parse_frontmatter(text)
|
||||
errors += [f"{rel}: {e}" for e in errs]
|
||||
if meta is None:
|
||||
continue
|
||||
for f in REQUIRED - meta.keys():
|
||||
errors.append(f"{rel}: 缺少必填字段 {f}")
|
||||
for f in LIST_FIELDS & meta.keys():
|
||||
if not isinstance(meta[f], list):
|
||||
errors.append(f"{rel}: 字段 {f} 必须是列表，例如 {f}: [a, b]")
|
||||
t = meta.get("type")
|
||||
if t not in TYPES:
|
||||
errors.append(f"{rel}: type={t!r} 不在 {sorted(TYPES)} 中")
|
||||
if meta.get("domain") not in DOMAINS:
|
||||
errors.append(f"{rel}: domain={meta.get('domain')!r} 不在 {sorted(DOMAINS)} 中")
|
||||
if not ID_RE.match(str(meta.get("id", ""))):
|
||||
errors.append(f"{rel}: id={meta.get('id')!r} 格式不对（小写字母数字，用 . 分段）")
|
||||
if t in ("lesson", "scenario", "interview") and meta.get("level") not in LEVELS:
|
||||
errors.append(f"{rel}: {t} 必须有 level（{sorted(LEVELS)}）")
|
||||
if "est_hours" in meta and not isinstance(meta["est_hours"], (int, float)):
|
||||
errors.append(f"{rel}: est_hours 必须是数字")
|
||||
for w in meta.get("week", []) if isinstance(meta.get("week"), list) else []:
|
||||
if not isinstance(w, int) or not 1 <= w <= 16:
|
||||
errors.append(f"{rel}: week 取值 {w!r} 必须是 1~16 的整数")
|
||||
closed, headings, links, counts = scan_body(body)
|
||||
if not closed:
|
||||
errors.append(f"{rel}: 代码块围栏没有闭合")
|
||||
for sec in REQUIRED_SECTIONS.get(t, []):
|
||||
if not any(sec in h for h in headings):
|
||||
errors.append(f"{rel}: {t} 缺少小节“## {sec}”")
|
||||
if t == "interview" and sum(counts.values()) == 0:
|
||||
errors.append(f"{rel}: 面试题文件没有题目（标题格式：### 【基础】问题）")
|
||||
for target in links:
|
||||
if re.match(r"^[a-z]+:", target) or target.startswith("#"):
|
||||
continue
|
||||
dest = (path.parent / target.split("#", 1)[0]).resolve()
|
||||
if not dest.exists():
|
||||
errors.append(f"{rel}: 链接指向的文件不存在：{target}")
|
||||
item = {
|
||||
"id": meta.get("id"), "title": meta.get("title"), "type": t, "domain": meta.get("domain"),
|
||||
"stack": meta.get("stack", []), "tags": meta.get("tags", []), "level": meta.get("level"),
|
||||
"prerequisites": meta.get("prerequisites", []), "related": meta.get("related", []),
|
||||
"est_hours": meta.get("est_hours"), "week": meta.get("week", []), "order": meta.get("order"),
|
||||
"path": rel, "sections": headings,
|
||||
}
|
||||
if t == "interview":
|
||||
item["questions"] = counts
|
||||
items.append(item)
|
||||
|
||||
ids = {}
|
||||
for it in items:
|
||||
if it["id"] in ids:
|
||||
errors.append(f"{it['path']}: id {it['id']} 与 {ids[it['id']]} 重复")
|
||||
ids[it["id"]] = it["path"]
|
||||
for it in items:
|
||||
for f in ("prerequisites", "related"):
|
||||
for ref in it[f] if isinstance(it[f], list) else []:
|
||||
if ref not in ids:
|
||||
errors.append(f"{it['path']}: {f} 引用了不存在的 id：{ref}")
|
||||
|
||||
stacks = []
|
||||
for it in items:
|
||||
if it["type"] == "index" and it["path"].startswith("topics/") and it["path"].count("/") == 3:
|
||||
sid = it["stack"][0] if it["stack"] else None
|
||||
mine = [x for x in items if x["path"].startswith(it["path"].rsplit("/", 1)[0] + "/")]
|
||||
q = {k: 0 for k in QUESTION_LEVELS}
|
||||
for x in mine:
|
||||
for k, v in x.get("questions", {}).items():
|
||||
q[k] += v
|
||||
stack_dir = ROOT / it["path"].rsplit("/", 1)[0]
|
||||
examples = sorted(p.relative_to(ROOT).as_posix() for p in (stack_dir / "examples").glob("*")) \
|
||||
if (stack_dir / "examples").exists() else []
|
||||
entry = {"stack": sid, "domain": it["domain"], "title": it["title"], "index": it["path"],
|
||||
"lessons": sum(x["type"] == "lesson" for x in mine),
|
||||
"scenarios": sum(x["type"] == "scenario" for x in mine),
|
||||
"interview_questions": q, "examples": examples}
|
||||
stacks.append(entry)
|
||||
if strict and (entry["lessons"] == 0 or entry["scenarios"] == 0 or sum(q.values()) == 0):
|
||||
errors.append(f"{it['path']}: 严格模式要求技术栈 {sid} 至少有 1 课、1 个场景和面试题")
|
||||
stacks.sort(key=lambda s: (s["domain"], s["stack"] or ""))
|
||||
|
||||
if errors:
|
||||
print(f"校验失败：{len(errors)} 个问题")
|
||||
for e in errors:
|
||||
print(" -", e)
|
||||
return 1
|
||||
q_total = {k: sum(s["interview_questions"][k] for s in stacks) for k in QUESTION_LEVELS}
|
||||
print(f"校验通过：{len(items)} 个文件，{len(stacks)} 个技术栈，"
|
||||
f"{sum(i['type'] == 'lesson' for i in items)} 课，{sum(i['type'] == 'scenario' for i in items)} 个场景，"
|
||||
f"面试题 {sum(q_total.values())} 道（{'、'.join(f'{k} {v}' for k, v in q_total.items())}）")
|
||||
if not check_only:
|
||||
catalog = {"schema_version": SCHEMA_VERSION, "generated_by": "tools/build_catalog.py",
|
||||
"stacks": stacks, "items": sorted(items, key=lambda x: x["id"])}
|
||||
(ROOT / "catalog.json").write_text(json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("已写出 catalog.json")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user