内容结构化改造（阶段 1）：新增 topics/scenarios 目录、内容规范、frontmatter schema、校验与目录生成脚本、31 个技术栈首页骨架，周文件加 frontmatter

tools/build_catalog.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""内容校验 + 目录生成（只用标准库，Python 3.10+）。
+
+用法：
+  python3 tools/build_catalog.py            # 校验并写出 catalog.json
+  python3 tools/build_catalog.py --check    # 只校验，不写文件（CI / 提交前用）
+  python3 tools/build_catalog.py --strict   # 额外要求：每个技术栈至少 1 课、1 个场景、1 份面试题
+
+校验内容：frontmatter 字段和取值、id 唯一、prerequisites / related 指向存在的 id、
+相对链接指向存在的文件、代码块围栏闭合、各类型必需的小节标题。规范见 CONTRIBUTING.md。
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+CONTENT_DIRS = ["plan", "topics", "scenarios"]
+SCHEMA_VERSION = 1
+
+TYPES = {"index", "lesson", "scenario", "interview", "project", "week"}
+DOMAINS = {"backend", "devops", "python", "ai", "frontend", "cross", "plan"}
+LEVELS = {"basic", "intermediate", "advanced"}
+QUESTION_LEVELS = ["基础", "进阶", "场景题"]
+ID_RE = re.compile(r"^[a-z0-9]+(?:\.[a-z0-9-]+)+$")
+REQUIRED = {"id", "title", "type", "domain"}
+LIST_FIELDS = {"stack", "tags", "prerequisites", "related", "week"}
+REQUIRED_SECTIONS = {
+    "lesson": ["解释说明", "案例", "代码示例", "易错点"],
+    "scenario": ["业务背景", "约束", "方案设计", "关键代码", "取舍", "易错点", "面试怎么讲"],
+}
+FENCE_RE = re.compile(r"^\s*(```|~~~)")
+LINK_RE = re.compile(r"(?<!!)\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
+QUESTION_RE = re.compile(r"^###\s+【(基础|进阶|场景题)】")
+
+
+def parse_scalar(v: str):
+    v = v.strip()
+    if v == "":
+        return ""
+    if (v[0] == v[-1]) and v[0] in "\"'" and len(v) >= 2:
+        return v[1:-1]
+    if re.fullmatch(r"-?\d+", v):
+        return int(v)
+    if re.fullmatch(r"-?\d+\.\d+", v):
+        return float(v)
+    if v in ("true", "false"):
+        return v == "true"
+    return v
+
+
+def parse_frontmatter(text: str) -> tuple[dict | None, str, list[str]]:
+    """只支持规范里约定的子集：`键: 标量` 和 `键: [a, b]`（单行流式列表）。"""
+    errs: list[str] = []
+    if not text.startswith("---\n"):
+        return None, text, ["缺少 frontmatter（文件第一行必须是 ---）"]
+    end = text.find("\n---\n", 4)
+    if end < 0:
+        return None, text, ["frontmatter 没有结束的 ---"]
+    meta: dict = {}
+    for n, line in enumerate(text[4:end].splitlines(), 2):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        m = re.match(r"^([a-z_]+):\s*(.*)$", line)
+        if not m:
+            errs.append(f"frontmatter 第 {n} 行无法解析：{line!r}")
+            continue
+        k, v = m.group(1), m.group(2).strip()
+        if v.startswith("["):
+            if not v.endswith("]"):
+                errs.append(f"frontmatter 字段 {k} 的列表必须写在一行内")
+                continue
+            inner = v[1:-1].strip()
+            meta[k] = [parse_scalar(x) for x in inner.split(",")] if inner else []
+        else:
+            meta[k] = parse_scalar(v)
+    return meta, text[end + 5:], errs
+
+
+def scan_body(body: str) -> tuple[bool, list[str], list[str], dict[str, int]]:
+    """返回：围栏是否闭合、二级标题、链接目标、按级别统计的面试题数。"""
+    in_fence, fence_mark = False, ""
+    headings, links = [], []
+    counts = {k: 0 for k in QUESTION_LEVELS}
+    for line in body.splitlines():
+        m = FENCE_RE.match(line)
+        if m:
+            if not in_fence:
+                in_fence, fence_mark = True, m.group(1)
+            elif m.group(1) == fence_mark and line.strip() == fence_mark:
+                in_fence = False
+            continue
+        if in_fence:
+            continue
+        if line.startswith("## "):
+            headings.append(line[3:].strip())
+        q = QUESTION_RE.match(line)
+        if q:
+            counts[q.group(1)] += 1
+        links += LINK_RE.findall(line)
+    return not in_fence, headings, links, counts
+
+
+def main() -> int:
+    check_only = "--check" in sys.argv
+    strict = "--strict" in sys.argv
+    errors: list[str] = []
+    items: list[dict] = []
+    files = sorted(p for d in CONTENT_DIRS for p in (ROOT / d).rglob("*.md")
+                   if "examples" not in p.relative_to(ROOT).parts and "node_modules" not in p.parts)
+    for path in files:
+        rel = path.relative_to(ROOT).as_posix()
+        text = path.read_text(encoding="utf-8")
+        meta, body, errs = parse_frontmatter(text)
+        errors += [f"{rel}: {e}" for e in errs]
+        if meta is None:
+            continue
+        for f in REQUIRED - meta.keys():
+            errors.append(f"{rel}: 缺少必填字段 {f}")
+        for f in LIST_FIELDS & meta.keys():
+            if not isinstance(meta[f], list):
+                errors.append(f"{rel}: 字段 {f} 必须是列表，例如 {f}: [a, b]")
+        t = meta.get("type")
+        if t not in TYPES:
+            errors.append(f"{rel}: type={t!r} 不在 {sorted(TYPES)} 中")
+        if meta.get("domain") not in DOMAINS:
+            errors.append(f"{rel}: domain={meta.get('domain')!r} 不在 {sorted(DOMAINS)} 中")
+        if not ID_RE.match(str(meta.get("id", ""))):
+            errors.append(f"{rel}: id={meta.get('id')!r} 格式不对（小写字母数字，用 . 分段）")
+        if t in ("lesson", "scenario", "interview") and meta.get("level") not in LEVELS:
+            errors.append(f"{rel}: {t} 必须有 level（{sorted(LEVELS)}）")
+        if "est_hours" in meta and not isinstance(meta["est_hours"], (int, float)):
+            errors.append(f"{rel}: est_hours 必须是数字")
+        for w in meta.get("week", []) if isinstance(meta.get("week"), list) else []:
+            if not isinstance(w, int) or not 1 <= w <= 16:
+                errors.append(f"{rel}: week 取值 {w!r} 必须是 1~16 的整数")
+        closed, headings, links, counts = scan_body(body)
+        if not closed:
+            errors.append(f"{rel}: 代码块围栏没有闭合")
+        for sec in REQUIRED_SECTIONS.get(t, []):
+            if not any(sec in h for h in headings):
+                errors.append(f"{rel}: {t} 缺少小节“## {sec}”")
+        if t == "interview" and sum(counts.values()) == 0:
+            errors.append(f"{rel}: 面试题文件没有题目（标题格式：### 【基础】问题）")
+        for target in links:
+            if re.match(r"^[a-z]+:", target) or target.startswith("#"):
+                continue
+            dest = (path.parent / target.split("#", 1)[0]).resolve()
+            if not dest.exists():
+                errors.append(f"{rel}: 链接指向的文件不存在：{target}")
+        item = {
+            "id": meta.get("id"), "title": meta.get("title"), "type": t, "domain": meta.get("domain"),
+            "stack": meta.get("stack", []), "tags": meta.get("tags", []), "level": meta.get("level"),
+            "prerequisites": meta.get("prerequisites", []), "related": meta.get("related", []),
+            "est_hours": meta.get("est_hours"), "week": meta.get("week", []), "order": meta.get("order"),
+            "path": rel, "sections": headings,
+        }
+        if t == "interview":
+            item["questions"] = counts
+        items.append(item)
+
+    ids = {}
+    for it in items:
+        if it["id"] in ids:
+            errors.append(f"{it['path']}: id {it['id']} 与 {ids[it['id']]} 重复")
+        ids[it["id"]] = it["path"]
+    for it in items:
+        for f in ("prerequisites", "related"):
+            for ref in it[f] if isinstance(it[f], list) else []:
+                if ref not in ids:
+                    errors.append(f"{it['path']}: {f} 引用了不存在的 id：{ref}")
+
+    stacks = []
+    for it in items:
+        if it["type"] == "index" and it["path"].startswith("topics/") and it["path"].count("/") == 3:
+            sid = it["stack"][0] if it["stack"] else None
+            mine = [x for x in items if x["path"].startswith(it["path"].rsplit("/", 1)[0] + "/")]
+            q = {k: 0 for k in QUESTION_LEVELS}
+            for x in mine:
+                for k, v in x.get("questions", {}).items():
+                    q[k] += v
+            stack_dir = ROOT / it["path"].rsplit("/", 1)[0]
+            examples = sorted(p.relative_to(ROOT).as_posix() for p in (stack_dir / "examples").glob("*")) \
+                if (stack_dir / "examples").exists() else []
+            entry = {"stack": sid, "domain": it["domain"], "title": it["title"], "index": it["path"],
+                     "lessons": sum(x["type"] == "lesson" for x in mine),
+                     "scenarios": sum(x["type"] == "scenario" for x in mine),
+                     "interview_questions": q, "examples": examples}
+            stacks.append(entry)
+            if strict and (entry["lessons"] == 0 or entry["scenarios"] == 0 or sum(q.values()) == 0):
+                errors.append(f"{it['path']}: 严格模式要求技术栈 {sid} 至少有 1 课、1 个场景和面试题")
+    stacks.sort(key=lambda s: (s["domain"], s["stack"] or ""))
+
+    if errors:
+        print(f"校验失败：{len(errors)} 个问题")
+        for e in errors:
+            print("  -", e)
+        return 1
+    q_total = {k: sum(s["interview_questions"][k] for s in stacks) for k in QUESTION_LEVELS}
+    print(f"校验通过：{len(items)} 个文件，{len(stacks)} 个技术栈，"
+          f"{sum(i['type'] == 'lesson' for i in items)} 课，{sum(i['type'] == 'scenario' for i in items)} 个场景，"
+          f"面试题 {sum(q_total.values())} 道（{'、'.join(f'{k} {v}' for k, v in q_total.items())}）")
+    if not check_only:
+        catalog = {"schema_version": SCHEMA_VERSION, "generated_by": "tools/build_catalog.py",
+                   "stacks": stacks, "items": sorted(items, key=lambda x: x["id"])}
+        (ROOT / "catalog.json").write_text(json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+        print("已写出 catalog.json")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/check_sensitive.py

@@ -0,0 +1,67 @@
+# tools/check_sensitive.py —— 提交前扫描暂存区内容
+# 不自动运行：作为 pre-commit 钩子在仓库根目录执行；加 --self-test 可以单独自检
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+PATTERNS = {
+    "内网 IP": re.compile(r"\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b|\b192\.168\.\d{1,3}\.\d{1,3}\b|\b172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}\b"),
+    "疑似密钥": re.compile(r"(?i)\b(?:sk|ak|api[_-]?key|secret|token)[\w-]*\s*[:=]\s*['\"]?[A-Za-z0-9_\-]{16,}"),
+    "私钥": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
+    "root 绝对路径": re.compile(r"(?<![\w/])/root/\S+"),  # sensitive-ok
+}
+WORDS_FILE = Path("notes/private/sensitive-words.txt")
+ALLOW_MARK = "sensitive-ok"  # 行内写这个标记表示人工确认过，可以放行（比如文档里讲解正则的例子）
+
+
+def load_words() -> list[str]:
+    if not WORDS_FILE.exists():
+        return []
+    return [w.strip() for w in WORDS_FILE.read_text(encoding="utf-8").splitlines() if w.strip() and not w.startswith("#")]
+
+
+def scan_text(name: str, text: str, words: list[str]) -> list[str]:
+    problems = []
+    for no, line in enumerate(text.splitlines(), start=1):
+        if ALLOW_MARK in line:
+            continue
+        for label, pat in PATTERNS.items():
+            if pat.search(line):
+                problems.append(f"{name}:{no} {label}")
+        for w in words:
+            if w.lower() in line.lower():
+                problems.append(f"{name}:{no} 敏感词（见私有清单）")  # 不打印敏感词本身
+    return problems
+
+
+def staged_files() -> list[str]:
+    out = subprocess.run(["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"],
+                         capture_output=True, text=True, check=True).stdout
+    return [f for f in out.splitlines() if f]
+
+
+def main() -> int:
+    words = load_words()
+    problems = []
+    for f in staged_files():
+        content = subprocess.run(["git", "show", f":{f}"], capture_output=True).stdout  # 读暂存区版本
+        try:
+            problems += scan_text(f, content.decode("utf-8"), words)
+        except UnicodeDecodeError:
+            continue  # 跳过二进制文件
+    for p in problems:
+        print("✗", p)
+    if problems:
+        print("发现疑似敏感内容，已阻止提交。确认无误的行可加注释 sensitive-ok。")
+    return 1 if problems else 0
+
+
+if __name__ == "__main__" and "--self-test" in sys.argv:
+    sample = "url: http://192.168.1.20:8080\napi_key = 'abcdefghijklmnopqrstuvwxyz'\n正常的一行\n"  # sensitive-ok
+    found = scan_text("demo.md", sample, ["某客户"])
+    assert len(found) == 2, found
+    assert scan_text("demo.md", "某客户的需求", ["某客户"]) == ["demo.md:1 敏感词（见私有清单）"]
+    print("自检通过：", found)
+elif __name__ == "__main__":
+    sys.exit(main())

tools/run_examples.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""运行 topics/**/examples/ 下的示例代码（抽查“代码能跑”）。
+
+- Go：含 go.mod 的目录执行 `go vet ./... && go test ./...`
+- Python：*.py 直接运行（文件开头可写 `# requires: 包1, 包2`，缺包时跳过）；test_*.py 用 pytest 运行
+- TypeScript：*.ts 用 Node.js 直接运行（需要 Node 23.6+ 的类型剥离，推荐 Node 24 LTS）
+- 文件开头写 `# 不自动运行` / `// 不自动运行` 的跳过（需要外部服务的示例）
+
+用法：python3 tools/run_examples.py [路径过滤关键词]
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def run(cmd: list[str] | str, cwd: Path, timeout: int = 180) -> tuple[bool, str]:
+    try:
+        p = subprocess.run(cmd, cwd=cwd, shell=isinstance(cmd, str), capture_output=True, text=True, timeout=timeout,
+                           env={**os.environ, "PYTHONPATH": str(cwd)})
+        return p.returncode == 0, (p.stdout + p.stderr)[-1500:]
+    except subprocess.TimeoutExpired:
+        return False, "超时"
+
+
+def node_ok() -> bool:
+    if not shutil.which("node"):
+        return False
+    v = subprocess.run(["node", "-v"], capture_output=True, text=True).stdout.strip().lstrip("v").split(".")
+    return (int(v[0]), int(v[1])) >= (23, 6)
+
+
+def main() -> int:
+    flt = sys.argv[1] if len(sys.argv) > 1 else ""
+    results: list[tuple[str, str, str]] = []
+    has_node = node_ok()
+    for ex in sorted(ROOT.glob("topics/**/examples/*")):
+        rel = ex.relative_to(ROOT).as_posix()
+        if flt and flt not in rel:
+            continue
+        if ex.is_dir() and (ex / "go.mod").exists():
+            if not shutil.which("go"):
+                results.append(("SKIP", rel, "没有安装 Go"))
+                continue
+            ok, out = run("go vet ./... && go test ./...", ex)
+            results.append(("PASS" if ok else "FAIL", rel, "" if ok else out))
+            continue
+        targets = [ex] if ex.is_file() else sorted(ex.rglob("*"))
+        for f in targets:
+            frel = f.relative_to(ROOT).as_posix()
+            if f.suffix not in (".py", ".ts") or "node_modules" in f.parts:
+                continue
+            head = f.read_text(encoding="utf-8")[:400]
+            if "不自动运行" in head:
+                results.append(("SKIP", frel, "标记为不自动运行"))
+                continue
+            if f.suffix == ".py":
+                m = re.search(r"^# requires:\s*(.+)$", head, re.M)
+                missing = [p.strip() for p in m.group(1).split(",")] if m else []
+                missing = [p for p in missing if importlib.util.find_spec(p) is None]
+                if missing:
+                    results.append(("SKIP", frel, f"缺少依赖 {missing}"))
+                    continue
+                cmd = [sys.executable, "-m", "pytest", "-q", f.name] if f.name.startswith("test_") \
+                    else [sys.executable, f.name]
+                ok, out = run(cmd, f.parent)
+            else:
+                if not has_node:
+                    results.append(("SKIP", frel, "需要 Node 23.6+"))
+                    continue
+                ok, out = run(["node", f.name], f.parent)
+            results.append(("PASS" if ok else "FAIL", frel, "" if ok else out))
+    for status, rel, out in results:
+        print(f"{status:4}  {rel}" + (f"\n{out}" if out and status == "FAIL" else f"  （{out}）" if out else ""))
+    n = {s: sum(r[0] == s for r in results) for s in ("PASS", "FAIL", "SKIP")}
+    print(f"\n通过 {n['PASS']}，失败 {n['FAIL']}，跳过 {n['SKIP']}")
+    return 1 if n["FAIL"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())