181 lines
8.1 KiB
Python
181 lines
8.1 KiB
Python
"""增量数据同步管道：第三方接口（限流 + 不稳定）→ 校验 → 幂等写入 → 数据质量检查 → 告警。
|
||
|
||
只用标准库（sqlite3 代替真实数据库）。`python pipeline.py` 直接运行演示，test_pipeline.py 是测试。
|
||
要点：
|
||
- 水位线（watermark）：记录已同步到的 (updated_at, id)，下次从这里继续，可断点续跑
|
||
- 限流：令牌桶控制调用频率，遇到 429 按 Retry-After 等待
|
||
- 重试：只重试可重试的错误，指数退避 + 抖动，有次数上限
|
||
- 幂等：按主键 upsert，并且只接受更新的版本，重复跑结果一样
|
||
- 坏数据进入死信表，不阻塞整批；质量检查不通过就告警并且不推进水位线
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import random
|
||
import sqlite3
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Callable, Iterable
|
||
|
||
|
||
class RetryableError(Exception):
|
||
"""网络超时、5xx、429 等，可以重试。"""
|
||
|
||
def __init__(self, msg: str, retry_after: float | None = None):
|
||
super().__init__(msg)
|
||
self.retry_after = retry_after
|
||
|
||
|
||
class FatalError(Exception):
|
||
"""401、参数错误等，重试没有意义。"""
|
||
|
||
|
||
class QualityError(Exception):
|
||
"""数据质量检查不通过。"""
|
||
|
||
|
||
@dataclass
|
||
class TokenBucket:
|
||
rate: float # 每秒补充的令牌数
|
||
capacity: float
|
||
now: Callable[[], float] = time.monotonic
|
||
sleep: Callable[[float], None] = time.sleep
|
||
tokens: float = field(init=False)
|
||
last: float = field(init=False)
|
||
|
||
def __post_init__(self) -> None:
|
||
self.tokens = self.capacity
|
||
self.last = self.now()
|
||
|
||
def acquire(self) -> None:
|
||
while True:
|
||
t = self.now()
|
||
self.tokens = min(self.capacity, self.tokens + (t - self.last) * self.rate)
|
||
self.last = t
|
||
if self.tokens >= 1:
|
||
self.tokens -= 1
|
||
return
|
||
self.sleep((1 - self.tokens) / self.rate)
|
||
|
||
|
||
def with_retry(fn: Callable[[], list[dict]], *, attempts: int = 5, base: float = 0.2, cap: float = 5.0,
|
||
sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None) -> list[dict]:
|
||
rng = rng or random.Random()
|
||
for i in range(attempts):
|
||
try:
|
||
return fn()
|
||
except RetryableError as e:
|
||
if i == attempts - 1:
|
||
raise
|
||
delay = e.retry_after if e.retry_after is not None else rng.uniform(0, min(cap, base * 2 ** i)) # 全抖动
|
||
sleep(delay)
|
||
raise AssertionError("unreachable")
|
||
|
||
|
||
SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS orders (id TEXT PRIMARY KEY, amount_cents INTEGER NOT NULL, status TEXT NOT NULL, updated_at TEXT NOT NULL);
|
||
CREATE TABLE IF NOT EXISTS dead_letters (source_id TEXT, reason TEXT, payload TEXT, at TEXT DEFAULT CURRENT_TIMESTAMP);
|
||
CREATE TABLE IF NOT EXISTS watermarks (job TEXT PRIMARY KEY, updated_at TEXT NOT NULL, last_id TEXT NOT NULL);
|
||
"""
|
||
VALID_STATUS = {"paid", "refunded", "cancelled"}
|
||
|
||
|
||
def validate(row: dict) -> str | None:
|
||
if not row.get("id"):
|
||
return "缺少 id"
|
||
if not isinstance(row.get("amount_cents"), int) or row["amount_cents"] < 0:
|
||
return "金额非法"
|
||
if row.get("status") not in VALID_STATUS:
|
||
return f"未知状态 {row.get('status')}"
|
||
if not row.get("updated_at"):
|
||
return "缺少 updated_at"
|
||
return None
|
||
|
||
|
||
@dataclass
|
||
class RunResult:
|
||
fetched: int = 0
|
||
written: int = 0
|
||
dead: int = 0
|
||
alerts: list[str] = field(default_factory=list)
|
||
committed: bool = False
|
||
|
||
|
||
def run_sync(conn: sqlite3.Connection, fetch_page: Callable[[str, str, int], list[dict]], *,
|
||
job: str = "orders", page_size: int = 100, bucket: TokenBucket | None = None,
|
||
max_dead_ratio: float = 0.05, alert: Callable[[str], None] = print,
|
||
sleep: Callable[[float], None] = time.sleep) -> RunResult:
|
||
"""从水位线开始分页拉取。fetch_page(after_updated_at, after_id, limit) 返回按 (updated_at, id) 升序的数据。"""
|
||
conn.executescript(SCHEMA)
|
||
wm = conn.execute("SELECT updated_at, last_id FROM watermarks WHERE job = ?", (job,)).fetchone()
|
||
after_ts, after_id = wm if wm else ("", "")
|
||
res = RunResult()
|
||
try:
|
||
with conn: # 整次运行一个事务：质量检查不过就整体回滚，水位线不前进
|
||
while True:
|
||
if bucket:
|
||
bucket.acquire()
|
||
page = with_retry(lambda: fetch_page(after_ts, after_id, page_size), sleep=sleep)
|
||
if not page:
|
||
break
|
||
for row in page:
|
||
res.fetched += 1
|
||
reason = validate(row)
|
||
if reason:
|
||
res.dead += 1
|
||
conn.execute("INSERT INTO dead_letters (source_id, reason, payload) VALUES (?, ?, ?)",
|
||
(str(row.get("id")), reason, repr(row)))
|
||
continue
|
||
# 幂等 upsert：只有来源版本更新时才覆盖，乱序或重复的数据不会把新值改回旧值
|
||
conn.execute(
|
||
"""INSERT INTO orders (id, amount_cents, status, updated_at) VALUES (:id, :amount_cents, :status, :updated_at)
|
||
ON CONFLICT(id) DO UPDATE SET amount_cents = excluded.amount_cents, status = excluded.status,
|
||
updated_at = excluded.updated_at
|
||
WHERE excluded.updated_at > orders.updated_at""", row)
|
||
res.written += 1
|
||
last = page[-1]
|
||
after_ts, after_id = last.get("updated_at") or after_ts, str(last.get("id") or after_id)
|
||
if len(page) < page_size:
|
||
break
|
||
# 数据质量检查
|
||
if res.fetched and res.dead / res.fetched > max_dead_ratio:
|
||
raise QualityError(f"坏数据比例 {res.dead}/{res.fetched} 超过阈值 {max_dead_ratio:.0%}")
|
||
conn.execute("INSERT INTO watermarks (job, updated_at, last_id) VALUES (?, ?, ?) "
|
||
"ON CONFLICT(job) DO UPDATE SET updated_at = excluded.updated_at, last_id = excluded.last_id",
|
||
(job, after_ts, after_id))
|
||
res.committed = True
|
||
except QualityError as e:
|
||
res.alerts.append(str(e))
|
||
alert(f"[告警] 任务 {job}：{e}，本次结果已回滚，水位线未推进")
|
||
return res
|
||
|
||
|
||
|
||
class FakeSource:
|
||
"""模拟第三方接口：按 (updated_at, id) 分页；可以注入 429 / 超时。"""
|
||
|
||
def __init__(self, rows: list[dict], failures: Iterable[Exception] = ()):
|
||
self.rows = sorted(rows, key=lambda r: (r["updated_at"], r["id"]))
|
||
self.failures = list(failures)
|
||
self.calls = 0
|
||
|
||
def __call__(self, after_ts: str, after_id: str, limit: int) -> list[dict]:
|
||
self.calls += 1
|
||
if self.failures:
|
||
raise self.failures.pop(0)
|
||
out = [r for r in self.rows if (r["updated_at"], r["id"]) > (after_ts, after_id)]
|
||
return [dict(r) for r in out[:limit]]
|
||
|
||
|
||
if __name__ == "__main__":
|
||
rows = [{"id": f"o{i:03d}", "amount_cents": 100 * i, "status": "paid", "updated_at": f"2026-10-23T10:{i // 60:02d}:{i % 60:02d}"} for i in range(250)]
|
||
rows.append({"id": "bad1", "amount_cents": -5, "status": "paid", "updated_at": "2026-10-23T11:00:00"})
|
||
src = FakeSource(rows, failures=[RetryableError("429 Too Many Requests", retry_after=0.01), RetryableError("timeout")])
|
||
conn = sqlite3.connect(":memory:")
|
||
r1 = run_sync(conn, src, page_size=100, bucket=TokenBucket(rate=50, capacity=5), sleep=lambda s: None)
|
||
print(f"第一次：拉取 {r1.fetched}，写入 {r1.written}，死信 {r1.dead}，接口调用 {src.calls} 次（含 2 次失败重试）")
|
||
r2 = run_sync(conn, src, page_size=100, sleep=lambda s: None)
|
||
print(f"第二次（没有新数据）：拉取 {r2.fetched}")
|
||
total = conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0]
|
||
print(f"orders 表 {total} 行，水位线 {conn.execute('SELECT updated_at, last_id FROM watermarks').fetchone()}")
|
||
assert (r1.written, r1.dead, r2.fetched, total) == (250, 1, 0, 250)
|