phase 5 part 3: mongodb, elasticsearch hybrid search, data-pipeline, nodejs BFF topics with verified examples

This commit is contained in:
Shikui Wang committed 2026-10-23 16:38:52 +08:00
1 parent 9eeb10aede
commit 075872e31e
27 files changed
+2405 -52

No files matched your search

@@ -0,0 +1,180 @@
"""增量数据同步管道:第三方接口(限流 + 不稳定)→ 校验 → 幂等写入 → 数据质量检查 → 告警。
只用标准库(sqlite3 代替真实数据库)。`python pipeline.py` 直接运行演示,test_pipeline.py 是测试。
要点:
- 水位线(watermark):记录已同步到的 (updated_at, id),下次从这里继续,可断点续跑
- 限流:令牌桶控制调用频率,遇到 429 按 Retry-After 等待
- 重试:只重试可重试的错误,指数退避 + 抖动,有次数上限
- 幂等:按主键 upsert,并且只接受更新的版本,重复跑结果一样
- 坏数据进入死信表,不阻塞整批;质量检查不通过就告警并且不推进水位线
"""
from __future__ import annotations
import random
import sqlite3
import time
from dataclasses import dataclass, field
from typing import Callable, Iterable
class RetryableError(Exception):
"""网络超时、5xx、429 等,可以重试。"""
def __init__(self, msg: str, retry_after: float | None = None):
super().__init__(msg)
self.retry_after = retry_after
class FatalError(Exception):
"""401、参数错误等,重试没有意义。"""
class QualityError(Exception):
"""数据质量检查不通过。"""
@dataclass
class TokenBucket:
rate: float # 每秒补充的令牌数
capacity: float
now: Callable[[], float] = time.monotonic
sleep: Callable[[float], None] = time.sleep
tokens: float = field(init=False)
last: float = field(init=False)
def __post_init__(self) -> None:
self.tokens = self.capacity
self.last = self.now()
def acquire(self) -> None:
while True:
t = self.now()
self.tokens = min(self.capacity, self.tokens + (t - self.last) * self.rate)
self.last = t
if self.tokens >= 1:
self.tokens -= 1
return
self.sleep((1 - self.tokens) / self.rate)
def with_retry(fn: Callable[[], list[dict]], *, attempts: int = 5, base: float = 0.2, cap: float = 5.0,
sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None) -> list[dict]:
rng = rng or random.Random()
for i in range(attempts):
try:
return fn()
except RetryableError as e:
if i == attempts - 1:
raise
delay = e.retry_after if e.retry_after is not None else rng.uniform(0, min(cap, base * 2 ** i)) # 全抖动
sleep(delay)
raise AssertionError("unreachable")
SCHEMA = """
CREATE TABLE IF NOT EXISTS orders (id TEXT PRIMARY KEY, amount_cents INTEGER NOT NULL, status TEXT NOT NULL, updated_at TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS dead_letters (source_id TEXT, reason TEXT, payload TEXT, at TEXT DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS watermarks (job TEXT PRIMARY KEY, updated_at TEXT NOT NULL, last_id TEXT NOT NULL);
"""
VALID_STATUS = {"paid", "refunded", "cancelled"}
def validate(row: dict) -> str | None:
if not row.get("id"):
return "缺少 id"
if not isinstance(row.get("amount_cents"), int) or row["amount_cents"] < 0:
return "金额非法"
if row.get("status") not in VALID_STATUS:
return f"未知状态 {row.get('status')}"
if not row.get("updated_at"):
return "缺少 updated_at"
return None
@dataclass
class RunResult:
fetched: int = 0
written: int = 0
dead: int = 0
alerts: list[str] = field(default_factory=list)
committed: bool = False
def run_sync(conn: sqlite3.Connection, fetch_page: Callable[[str, str, int], list[dict]], *,
job: str = "orders", page_size: int = 100, bucket: TokenBucket | None = None,
max_dead_ratio: float = 0.05, alert: Callable[[str], None] = print,
sleep: Callable[[float], None] = time.sleep) -> RunResult:
"""从水位线开始分页拉取。fetch_page(after_updated_at, after_id, limit) 返回按 (updated_at, id) 升序的数据。"""
conn.executescript(SCHEMA)
wm = conn.execute("SELECT updated_at, last_id FROM watermarks WHERE job = ?", (job,)).fetchone()
after_ts, after_id = wm if wm else ("", "")
res = RunResult()
try:
with conn: # 整次运行一个事务:质量检查不过就整体回滚,水位线不前进
while True:
if bucket:
bucket.acquire()
page = with_retry(lambda: fetch_page(after_ts, after_id, page_size), sleep=sleep)
if not page:
break
for row in page:
res.fetched += 1
reason = validate(row)
if reason:
res.dead += 1
conn.execute("INSERT INTO dead_letters (source_id, reason, payload) VALUES (?, ?, ?)",
(str(row.get("id")), reason, repr(row)))
continue
# 幂等 upsert:只有来源版本更新时才覆盖,乱序或重复的数据不会把新值改回旧值
conn.execute(
"""INSERT INTO orders (id, amount_cents, status, updated_at) VALUES (:id, :amount_cents, :status, :updated_at)
ON CONFLICT(id) DO UPDATE SET amount_cents = excluded.amount_cents, status = excluded.status,
updated_at = excluded.updated_at
WHERE excluded.updated_at > orders.updated_at""", row)
res.written += 1
last = page[-1]
after_ts, after_id = last.get("updated_at") or after_ts, str(last.get("id") or after_id)
if len(page) < page_size:
break
# 数据质量检查
if res.fetched and res.dead / res.fetched > max_dead_ratio:
raise QualityError(f"坏数据比例 {res.dead}/{res.fetched} 超过阈值 {max_dead_ratio:.0%}")
conn.execute("INSERT INTO watermarks (job, updated_at, last_id) VALUES (?, ?, ?) "
"ON CONFLICT(job) DO UPDATE SET updated_at = excluded.updated_at, last_id = excluded.last_id",
(job, after_ts, after_id))
res.committed = True
except QualityError as e:
res.alerts.append(str(e))
alert(f"[告警] 任务 {job}:{e},本次结果已回滚,水位线未推进")
return res
class FakeSource:
"""模拟第三方接口:按 (updated_at, id) 分页;可以注入 429 / 超时。"""
def __init__(self, rows: list[dict], failures: Iterable[Exception] = ()):
self.rows = sorted(rows, key=lambda r: (r["updated_at"], r["id"]))
self.failures = list(failures)
self.calls = 0
def __call__(self, after_ts: str, after_id: str, limit: int) -> list[dict]:
self.calls += 1
if self.failures:
raise self.failures.pop(0)
out = [r for r in self.rows if (r["updated_at"], r["id"]) > (after_ts, after_id)]
return [dict(r) for r in out[:limit]]
if __name__ == "__main__":
rows = [{"id": f"o{i:03d}", "amount_cents": 100 * i, "status": "paid", "updated_at": f"2026-10-23T10:{i // 60:02d}:{i % 60:02d}"} for i in range(250)]
rows.append({"id": "bad1", "amount_cents": -5, "status": "paid", "updated_at": "2026-10-23T11:00:00"})
src = FakeSource(rows, failures=[RetryableError("429 Too Many Requests", retry_after=0.01), RetryableError("timeout")])
conn = sqlite3.connect(":memory:")
r1 = run_sync(conn, src, page_size=100, bucket=TokenBucket(rate=50, capacity=5), sleep=lambda s: None)
print(f"第一次:拉取 {r1.fetched},写入 {r1.written},死信 {r1.dead},接口调用 {src.calls} 次(含 2 次失败重试)")
r2 = run_sync(conn, src, page_size=100, sleep=lambda s: None)
print(f"第二次(没有新数据):拉取 {r2.fetched}")
total = conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0]
print(f"orders 表 {total} 行,水位线 {conn.execute('SELECT updated_at, last_id FROM watermarks').fetchone()}")
assert (r1.written, r1.dead, r2.fetched, total) == (250, 1, 0, 250)