phase 5 part 3: mongodb, elasticsearch hybrid search, data-pipeline, nodejs BFF topics with verified examples

topics/backend/data-pipeline/examples/incremental-sync/pipeline.py

@@ -0,0 +1,180 @@
+"""增量数据同步管道：第三方接口（限流 + 不稳定）→ 校验 → 幂等写入 → 数据质量检查 → 告警。
+
+只用标准库（sqlite3 代替真实数据库）。`python pipeline.py` 直接运行演示，test_pipeline.py 是测试。
+要点：
+- 水位线（watermark）：记录已同步到的 (updated_at, id)，下次从这里继续，可断点续跑
+- 限流：令牌桶控制调用频率，遇到 429 按 Retry-After 等待
+- 重试：只重试可重试的错误，指数退避 + 抖动，有次数上限
+- 幂等：按主键 upsert，并且只接受更新的版本，重复跑结果一样
+- 坏数据进入死信表，不阻塞整批；质量检查不通过就告警并且不推进水位线
+"""
+from __future__ import annotations
+
+import random
+import sqlite3
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Iterable
+
+
+class RetryableError(Exception):
+    """网络超时、5xx、429 等，可以重试。"""
+
+    def __init__(self, msg: str, retry_after: float | None = None):
+        super().__init__(msg)
+        self.retry_after = retry_after
+
+
+class FatalError(Exception):
+    """401、参数错误等，重试没有意义。"""
+
+
+class QualityError(Exception):
+    """数据质量检查不通过。"""
+
+
+@dataclass
+class TokenBucket:
+    rate: float                     # 每秒补充的令牌数
+    capacity: float
+    now: Callable[[], float] = time.monotonic
+    sleep: Callable[[float], None] = time.sleep
+    tokens: float = field(init=False)
+    last: float = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.tokens = self.capacity
+        self.last = self.now()
+
+    def acquire(self) -> None:
+        while True:
+            t = self.now()
+            self.tokens = min(self.capacity, self.tokens + (t - self.last) * self.rate)
+            self.last = t
+            if self.tokens >= 1:
+                self.tokens -= 1
+                return
+            self.sleep((1 - self.tokens) / self.rate)
+
+
+def with_retry(fn: Callable[[], list[dict]], *, attempts: int = 5, base: float = 0.2, cap: float = 5.0,
+               sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None) -> list[dict]:
+    rng = rng or random.Random()
+    for i in range(attempts):
+        try:
+            return fn()
+        except RetryableError as e:
+            if i == attempts - 1:
+                raise
+            delay = e.retry_after if e.retry_after is not None else rng.uniform(0, min(cap, base * 2 ** i))  # 全抖动
+            sleep(delay)
+    raise AssertionError("unreachable")
+
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS orders (id TEXT PRIMARY KEY, amount_cents INTEGER NOT NULL, status TEXT NOT NULL, updated_at TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS dead_letters (source_id TEXT, reason TEXT, payload TEXT, at TEXT DEFAULT CURRENT_TIMESTAMP);
+CREATE TABLE IF NOT EXISTS watermarks (job TEXT PRIMARY KEY, updated_at TEXT NOT NULL, last_id TEXT NOT NULL);
+"""
+VALID_STATUS = {"paid", "refunded", "cancelled"}
+
+
+def validate(row: dict) -> str | None:
+    if not row.get("id"):
+        return "缺少 id"
+    if not isinstance(row.get("amount_cents"), int) or row["amount_cents"] < 0:
+        return "金额非法"
+    if row.get("status") not in VALID_STATUS:
+        return f"未知状态 {row.get('status')}"
+    if not row.get("updated_at"):
+        return "缺少 updated_at"
+    return None
+
+
+@dataclass
+class RunResult:
+    fetched: int = 0
+    written: int = 0
+    dead: int = 0
+    alerts: list[str] = field(default_factory=list)
+    committed: bool = False
+
+
+def run_sync(conn: sqlite3.Connection, fetch_page: Callable[[str, str, int], list[dict]], *,
+             job: str = "orders", page_size: int = 100, bucket: TokenBucket | None = None,
+             max_dead_ratio: float = 0.05, alert: Callable[[str], None] = print,
+             sleep: Callable[[float], None] = time.sleep) -> RunResult:
+    """从水位线开始分页拉取。fetch_page(after_updated_at, after_id, limit) 返回按 (updated_at, id) 升序的数据。"""
+    conn.executescript(SCHEMA)
+    wm = conn.execute("SELECT updated_at, last_id FROM watermarks WHERE job = ?", (job,)).fetchone()
+    after_ts, after_id = wm if wm else ("", "")
+    res = RunResult()
+    try:
+        with conn:                                   # 整次运行一个事务：质量检查不过就整体回滚，水位线不前进
+            while True:
+                if bucket:
+                    bucket.acquire()
+                page = with_retry(lambda: fetch_page(after_ts, after_id, page_size), sleep=sleep)
+                if not page:
+                    break
+                for row in page:
+                    res.fetched += 1
+                    reason = validate(row)
+                    if reason:
+                        res.dead += 1
+                        conn.execute("INSERT INTO dead_letters (source_id, reason, payload) VALUES (?, ?, ?)",
+                                     (str(row.get("id")), reason, repr(row)))
+                        continue
+                    # 幂等 upsert：只有来源版本更新时才覆盖，乱序或重复的数据不会把新值改回旧值
+                    conn.execute(
+                        """INSERT INTO orders (id, amount_cents, status, updated_at) VALUES (:id, :amount_cents, :status, :updated_at)
+                           ON CONFLICT(id) DO UPDATE SET amount_cents = excluded.amount_cents, status = excluded.status,
+                               updated_at = excluded.updated_at
+                           WHERE excluded.updated_at > orders.updated_at""", row)
+                    res.written += 1
+                last = page[-1]
+                after_ts, after_id = last.get("updated_at") or after_ts, str(last.get("id") or after_id)
+                if len(page) < page_size:
+                    break
+            # 数据质量检查
+            if res.fetched and res.dead / res.fetched > max_dead_ratio:
+                raise QualityError(f"坏数据比例 {res.dead}/{res.fetched} 超过阈值 {max_dead_ratio:.0%}")
+            conn.execute("INSERT INTO watermarks (job, updated_at, last_id) VALUES (?, ?, ?) "
+                         "ON CONFLICT(job) DO UPDATE SET updated_at = excluded.updated_at, last_id = excluded.last_id",
+                         (job, after_ts, after_id))
+            res.committed = True
+    except QualityError as e:
+        res.alerts.append(str(e))
+        alert(f"[告警] 任务 {job}：{e}，本次结果已回滚，水位线未推进")
+    return res
+
+
+
+class FakeSource:
+    """模拟第三方接口：按 (updated_at, id) 分页；可以注入 429 / 超时。"""
+
+    def __init__(self, rows: list[dict], failures: Iterable[Exception] = ()):
+        self.rows = sorted(rows, key=lambda r: (r["updated_at"], r["id"]))
+        self.failures = list(failures)
+        self.calls = 0
+
+    def __call__(self, after_ts: str, after_id: str, limit: int) -> list[dict]:
+        self.calls += 1
+        if self.failures:
+            raise self.failures.pop(0)
+        out = [r for r in self.rows if (r["updated_at"], r["id"]) > (after_ts, after_id)]
+        return [dict(r) for r in out[:limit]]
+
+
+if __name__ == "__main__":
+    rows = [{"id": f"o{i:03d}", "amount_cents": 100 * i, "status": "paid", "updated_at": f"2026-10-23T10:{i // 60:02d}:{i % 60:02d}"} for i in range(250)]
+    rows.append({"id": "bad1", "amount_cents": -5, "status": "paid", "updated_at": "2026-10-23T11:00:00"})
+    src = FakeSource(rows, failures=[RetryableError("429 Too Many Requests", retry_after=0.01), RetryableError("timeout")])
+    conn = sqlite3.connect(":memory:")
+    r1 = run_sync(conn, src, page_size=100, bucket=TokenBucket(rate=50, capacity=5), sleep=lambda s: None)
+    print(f"第一次：拉取 {r1.fetched}，写入 {r1.written}，死信 {r1.dead}，接口调用 {src.calls} 次（含 2 次失败重试）")
+    r2 = run_sync(conn, src, page_size=100, sleep=lambda s: None)
+    print(f"第二次（没有新数据）：拉取 {r2.fetched}")
+    total = conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0]
+    print(f"orders 表 {total} 行，水位线 {conn.execute('SELECT updated_at, last_id FROM watermarks').fetchone()}")
+    assert (r1.written, r1.dead, r2.fetched, total) == (250, 1, 0, 250)

topics/backend/data-pipeline/examples/incremental-sync/test_pipeline.py

@@ -0,0 +1,80 @@
+import sqlite3
+
+import pytest
+
+from pipeline import FakeSource, FatalError, RetryableError, TokenBucket, run_sync, with_retry
+
+
+def make_rows(n, start=0, ts_prefix="2026-10-23T10:00:", status="paid"):
+    return [{"id": f"o{i:03d}", "amount_cents": i, "status": status, "updated_at": f"{ts_prefix}{i:02d}"} for i in range(start, start + n)]
+
+
+def test_resume_from_watermark_and_idempotent():
+    conn = sqlite3.connect(":memory:")
+    src = FakeSource(make_rows(30))
+    assert run_sync(conn, src, page_size=10, sleep=lambda s: None).written == 30
+    again = run_sync(conn, src, page_size=10, sleep=lambda s: None)
+    assert again.fetched == 0                                       # 水位线生效，不重复拉取
+    src.rows += make_rows(5, start=30)
+    assert run_sync(conn, src, page_size=10, sleep=lambda s: None).written == 5
+    assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 35
+
+
+def test_older_version_does_not_overwrite_newer():
+    conn = sqlite3.connect(":memory:")
+    run_sync(conn, FakeSource([{"id": "a", "amount_cents": 1, "status": "refunded", "updated_at": "2026-10-23T12:00:00"}]), job="j1")
+    # 另一个来源（或重放）送来旧版本
+    run_sync(conn, FakeSource([{"id": "a", "amount_cents": 1, "status": "paid", "updated_at": "2026-10-23T09:00:00"}]), job="j2")
+    assert conn.execute("SELECT status FROM orders WHERE id='a'").fetchone()[0] == "refunded"
+
+
+def test_quality_gate_rolls_back_and_alerts():
+    conn = sqlite3.connect(":memory:")
+    rows = make_rows(10) + make_rows(3, start=10, status="???")    # 3/13 坏数据，超过 5%
+    alerts = []
+    r = run_sync(conn, FakeSource(rows), max_dead_ratio=0.05, alert=alerts.append)
+    assert not r.committed and alerts
+    assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 0           # 整批回滚
+    assert conn.execute("SELECT COUNT(*) FROM watermarks").fetchone()[0] == 0       # 水位线没动
+
+
+def test_retry_only_retryable_and_respects_retry_after():
+    sleeps = []
+    calls = iter([RetryableError("429", retry_after=2.0), RetryableError("timeout"), ["ok"]])
+
+    def fn():
+        v = next(calls)
+        if isinstance(v, Exception):
+            raise v
+        return v
+
+    assert with_retry(fn, sleep=sleeps.append) == ["ok"]
+    assert sleeps[0] == 2.0 and 0 <= sleeps[1] <= 0.4
+
+    def fatal():
+        raise FatalError("401")
+
+    with pytest.raises(FatalError):
+        with_retry(fatal, sleep=sleeps.append)
+
+
+def test_retry_gives_up():
+    def always():
+        raise RetryableError("503")
+
+    with pytest.raises(RetryableError):
+        with_retry(always, attempts=3, sleep=lambda s: None)
+
+
+def test_token_bucket_limits_rate():
+    clock = [0.0]
+    slept = []
+
+    def sleep(s):
+        slept.append(s)
+        clock[0] += s
+
+    b = TokenBucket(rate=2, capacity=2, now=lambda: clock[0], sleep=sleep)
+    for _ in range(6):
+        b.acquire()
+    assert clock[0] == pytest.approx(2.0)            # 2 个初始令牌 + 4 个以每秒 2 个的速度补充 = 2 秒