"""增量数据同步管道：第三方接口（限流 + 不稳定）→ 校验 → 幂等写入 → 数据质量检查 → 告警。 只用标准库（sqlite3 代替真实数据库）。`python pipeline.py` 直接运行演示，test_pipeline.py 是测试。 要点： - 水位线（watermark）：记录已同步到的 (updated_at, id)，下次从这里继续，可断点续跑 - 限流：令牌桶控制调用频率，遇到 429 按 Retry-After 等待 - 重试：只重试可重试的错误，指数退避 + 抖动，有次数上限 - 幂等：按主键 upsert，并且只接受更新的版本，重复跑结果一样 - 坏数据进入死信表，不阻塞整批；质量检查不通过就告警并且不推进水位线 """ from __future__ import annotations import random import sqlite3 import time from dataclasses import dataclass, field from typing import Callable, Iterable class RetryableError(Exception): """网络超时、5xx、429 等，可以重试。""" def __init__(self, msg: str, retry_after: float | None = None): super().__init__(msg) self.retry_after = retry_after class FatalError(Exception): """401、参数错误等，重试没有意义。""" class QualityError(Exception): """数据质量检查不通过。""" @dataclass class TokenBucket: rate: float # 每秒补充的令牌数 capacity: float now: Callable[[], float] = time.monotonic sleep: Callable[[float], None] = time.sleep tokens: float = field(init=False) last: float = field(init=False) def __post_init__(self) -> None: self.tokens = self.capacity self.last = self.now() def acquire(self) -> None: while True: t = self.now() self.tokens = min(self.capacity, self.tokens + (t - self.last) * self.rate) self.last = t if self.tokens >= 1: self.tokens -= 1 return self.sleep((1 - self.tokens) / self.rate) def with_retry(fn: Callable[[], list[dict]], *, attempts: int = 5, base: float = 0.2, cap: float = 5.0, sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None) -> list[dict]: rng = rng or random.Random() for i in range(attempts): try: return fn() except RetryableError as e: if i == attempts - 1: raise delay = e.retry_after if e.retry_after is not None else rng.uniform(0, min(cap, base * 2 ** i)) # 全抖动 sleep(delay) raise AssertionError("unreachable") SCHEMA = """ CREATE TABLE IF NOT EXISTS orders (id TEXT PRIMARY KEY, amount_cents INTEGER NOT NULL, status TEXT NOT NULL, updated_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS dead_letters (source_id TEXT, reason TEXT, payload TEXT, at TEXT DEFAULT CURRENT_TIMESTAMP); CREATE TABLE IF NOT EXISTS watermarks (job TEXT PRIMARY KEY, updated_at TEXT NOT NULL, last_id TEXT NOT NULL); """ VALID_STATUS = {"paid", "refunded", "cancelled"} def validate(row: dict) -> str | None: if not row.get("id"): return "缺少 id" if not isinstance(row.get("amount_cents"), int) or row["amount_cents"] < 0: return "金额非法" if row.get("status") not in VALID_STATUS: return f"未知状态 {row.get('status')}" if not row.get("updated_at"): return "缺少 updated_at" return None @dataclass class RunResult: fetched: int = 0 written: int = 0 dead: int = 0 alerts: list[str] = field(default_factory=list) committed: bool = False def run_sync(conn: sqlite3.Connection, fetch_page: Callable[[str, str, int], list[dict]], *, job: str = "orders", page_size: int = 100, bucket: TokenBucket | None = None, max_dead_ratio: float = 0.05, alert: Callable[[str], None] = print, sleep: Callable[[float], None] = time.sleep) -> RunResult: """从水位线开始分页拉取。fetch_page(after_updated_at, after_id, limit) 返回按 (updated_at, id) 升序的数据。""" conn.executescript(SCHEMA) wm = conn.execute("SELECT updated_at, last_id FROM watermarks WHERE job = ?", (job,)).fetchone() after_ts, after_id = wm if wm else ("", "") res = RunResult() try: with conn: # 整次运行一个事务：质量检查不过就整体回滚，水位线不前进 while True: if bucket: bucket.acquire() page = with_retry(lambda: fetch_page(after_ts, after_id, page_size), sleep=sleep) if not page: break for row in page: res.fetched += 1 reason = validate(row) if reason: res.dead += 1 conn.execute("INSERT INTO dead_letters (source_id, reason, payload) VALUES (?, ?, ?)", (str(row.get("id")), reason, repr(row))) continue # 幂等 upsert：只有来源版本更新时才覆盖，乱序或重复的数据不会把新值改回旧值 conn.execute( """INSERT INTO orders (id, amount_cents, status, updated_at) VALUES (:id, :amount_cents, :status, :updated_at) ON CONFLICT(id) DO UPDATE SET amount_cents = excluded.amount_cents, status = excluded.status, updated_at = excluded.updated_at WHERE excluded.updated_at > orders.updated_at""", row) res.written += 1 last = page[-1] after_ts, after_id = last.get("updated_at") or after_ts, str(last.get("id") or after_id) if len(page) < page_size: break # 数据质量检查 if res.fetched and res.dead / res.fetched > max_dead_ratio: raise QualityError(f"坏数据比例 {res.dead}/{res.fetched} 超过阈值 {max_dead_ratio:.0%}") conn.execute("INSERT INTO watermarks (job, updated_at, last_id) VALUES (?, ?, ?) " "ON CONFLICT(job) DO UPDATE SET updated_at = excluded.updated_at, last_id = excluded.last_id", (job, after_ts, after_id)) res.committed = True except QualityError as e: res.alerts.append(str(e)) alert(f"[告警] 任务 {job}：{e}，本次结果已回滚，水位线未推进") return res class FakeSource: """模拟第三方接口：按 (updated_at, id) 分页；可以注入 429 / 超时。""" def __init__(self, rows: list[dict], failures: Iterable[Exception] = ()): self.rows = sorted(rows, key=lambda r: (r["updated_at"], r["id"])) self.failures = list(failures) self.calls = 0 def __call__(self, after_ts: str, after_id: str, limit: int) -> list[dict]: self.calls += 1 if self.failures: raise self.failures.pop(0) out = [r for r in self.rows if (r["updated_at"], r["id"]) > (after_ts, after_id)] return [dict(r) for r in out[:limit]] if __name__ == "__main__": rows = [{"id": f"o{i:03d}", "amount_cents": 100 * i, "status": "paid", "updated_at": f"2026-10-23T10:{i // 60:02d}:{i % 60:02d}"} for i in range(250)] rows.append({"id": "bad1", "amount_cents": -5, "status": "paid", "updated_at": "2026-10-23T11:00:00"}) src = FakeSource(rows, failures=[RetryableError("429 Too Many Requests", retry_after=0.01), RetryableError("timeout")]) conn = sqlite3.connect(":memory:") r1 = run_sync(conn, src, page_size=100, bucket=TokenBucket(rate=50, capacity=5), sleep=lambda s: None) print(f"第一次：拉取 {r1.fetched}，写入 {r1.written}，死信 {r1.dead}，接口调用 {src.calls} 次（含 2 次失败重试）") r2 = run_sync(conn, src, page_size=100, sleep=lambda s: None) print(f"第二次（没有新数据）：拉取 {r2.fetched}") total = conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] print(f"orders 表 {total} 行，水位线 {conn.execute('SELECT updated_at, last_id FROM watermarks').fetchone()}") assert (r1.written, r1.dead, r2.fetched, total) == (250, 1, 0, 250)