阶段2(上):后端专题 Go/Gin/PostgreSQL/MySQL/Redis/MQ/gRPC 的课程、企业场景、面试题和可运行示例
This commit is contained in:
1 parent
0bae9916a9
commit
5eb67c5d08
65 files changed
+6457
-77
No files matched your search
@@ -0,0 +1,84 @@
|
||||
# requires: aio_pika
|
||||
"""RabbitMQ 延迟取消订单:消息先进带 TTL 的等待队列,过期后经死信交换机转到处理队列。
|
||||
|
||||
运行:AMQP_URL=amqp://guest:guest@localhost/ python delay_cancel.py
|
||||
没有设置 AMQP_URL 时打印提示并正常退出。会创建并删除名字带随机后缀的交换机和队列。
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aio_pika
|
||||
|
||||
DELAY_MS = 1000 # 演示用 1 秒;真实业务常见是 15~30 分钟
|
||||
|
||||
|
||||
class OrderStore:
|
||||
"""模拟数据库里的订单状态(真实项目里用条件 UPDATE 保证幂等)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.status: dict[str, str] = {}
|
||||
|
||||
def cancel_if_unpaid(self, order_id: str) -> bool:
|
||||
# 等价于 UPDATE orders SET status='cancelled' WHERE id=? AND status='pending'
|
||||
if self.status.get(order_id) == "pending":
|
||||
self.status[order_id] = "cancelled"
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
url = os.environ.get("AMQP_URL")
|
||||
if not url:
|
||||
print("未设置 AMQP_URL,跳过(示例:amqp://guest:guest@localhost/)")
|
||||
return 0
|
||||
suffix = uuid.uuid4().hex[:8]
|
||||
conn = await aio_pika.connect_robust(url)
|
||||
async with conn:
|
||||
ch = await conn.channel()
|
||||
await ch.set_qos(prefetch_count=10)
|
||||
dlx = await ch.declare_exchange(f"order.dlx.{suffix}", aio_pika.ExchangeType.DIRECT, auto_delete=True)
|
||||
# 处理队列:真正执行“超时取消”的消费者监听这里
|
||||
work_q = await ch.declare_queue(f"order.cancel.{suffix}", durable=True, auto_delete=True)
|
||||
await work_q.bind(dlx, routing_key="cancel")
|
||||
# 等待队列:没有消费者,消息在这里过期后转到死信交换机
|
||||
wait_q = await ch.declare_queue(f"order.wait.{suffix}", durable=True, auto_delete=True, arguments={
|
||||
"x-message-ttl": DELAY_MS,
|
||||
"x-dead-letter-exchange": dlx.name,
|
||||
"x-dead-letter-routing-key": "cancel",
|
||||
})
|
||||
|
||||
store = OrderStore()
|
||||
store.status = {"o1": "pending", "o2": "pending"}
|
||||
start = time.monotonic()
|
||||
for oid in ("o1", "o2", "o1"): # o1 故意发两次,模拟重复消息
|
||||
await ch.default_exchange.publish(
|
||||
aio_pika.Message(json.dumps({"order_id": oid}).encode(),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT, message_id=f"cancel-{oid}"),
|
||||
routing_key=wait_q.name)
|
||||
store.status["o2"] = "paid" # o2 在超时前支付了
|
||||
|
||||
results: list[tuple[str, bool]] = []
|
||||
async with work_q.iterator() as it:
|
||||
async for msg in it:
|
||||
async with msg.process(): # 正常结束自动 ack,抛异常则 nack
|
||||
oid = json.loads(msg.body)["order_id"]
|
||||
results.append((oid, store.cancel_if_unpaid(oid)))
|
||||
if len(results) == 3:
|
||||
break
|
||||
elapsed = time.monotonic() - start
|
||||
print("处理结果:", results, f"耗时 {elapsed:.1f}s")
|
||||
assert elapsed >= DELAY_MS / 1000 * 0.9, "消息应该在延迟之后才被处理"
|
||||
assert store.status == {"o1": "cancelled", "o2": "paid"}, store.status
|
||||
assert sorted(results) == [("o1", False), ("o1", True), ("o2", False)] # 重复消息不会重复取消
|
||||
await wait_q.delete(if_unused=False, if_empty=False)
|
||||
await work_q.delete(if_unused=False, if_empty=False)
|
||||
print("延迟取消正确:未支付的被取消,已支付的不受影响,重复消息幂等")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,105 @@
|
||||
"""发件箱(Outbox)模式:业务数据和“待发送消息”写在同一个数据库事务里,由后台程序投递。
|
||||
|
||||
只用标准库(sqlite3)演示,直接运行:python outbox_demo.py
|
||||
真实项目里把 sqlite 换成 MySQL / PostgreSQL,把 FakeBroker 换成 RabbitMQ / Kafka 客户端。
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
|
||||
class FakeBroker:
|
||||
"""模拟消息队列:fail_times 次之内发送失败,用来演示重试。"""
|
||||
|
||||
def __init__(self, fail_times: int = 0) -> None:
|
||||
self.sent: list[dict] = []
|
||||
self.fail_times = fail_times
|
||||
|
||||
def publish(self, topic: str, msg: dict) -> None:
|
||||
if self.fail_times > 0:
|
||||
self.fail_times -= 1
|
||||
raise ConnectionError("broker 暂时不可用")
|
||||
self.sent.append({"topic": topic, **msg})
|
||||
|
||||
|
||||
def init(db: sqlite3.Connection) -> None:
|
||||
db.executescript("""
|
||||
CREATE TABLE orders (id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, amount INTEGER NOT NULL);
|
||||
CREATE TABLE outbox (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
topic TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
sent_at TEXT, -- NULL 表示还没发送
|
||||
attempts INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
""")
|
||||
|
||||
|
||||
def create_order(db: sqlite3.Connection, user_id: int, amount: int, fail_after_insert: bool = False) -> int:
|
||||
with db: # 一个事务:订单和消息要么都写入,要么都不写入
|
||||
cur = db.execute("INSERT INTO orders (user_id, amount) VALUES (?, ?)", (user_id, amount))
|
||||
order_id = cur.lastrowid
|
||||
db.execute("INSERT INTO outbox (topic, payload) VALUES (?, ?)",
|
||||
("order.created", json.dumps({"order_id": order_id, "user_id": user_id, "amount": amount})))
|
||||
if fail_after_insert:
|
||||
raise RuntimeError("模拟业务异常:事务回滚")
|
||||
return order_id
|
||||
|
||||
|
||||
def relay(db: sqlite3.Connection, broker: FakeBroker, batch: int = 100) -> int:
|
||||
"""后台投递:取未发送的消息,发送成功后标记。可能重复发送(至少一次),消费者要幂等。"""
|
||||
rows = db.execute("SELECT id, topic, payload FROM outbox WHERE sent_at IS NULL ORDER BY id LIMIT ?",
|
||||
(batch,)).fetchall()
|
||||
sent = 0
|
||||
for mid, topic, payload in rows:
|
||||
try:
|
||||
broker.publish(topic, {"message_id": mid, **json.loads(payload)})
|
||||
except ConnectionError:
|
||||
with db:
|
||||
db.execute("UPDATE outbox SET attempts = attempts + 1 WHERE id = ?", (mid,))
|
||||
break # 保持顺序:前面的没发出去,后面的先不发
|
||||
with db:
|
||||
db.execute("UPDATE outbox SET sent_at = datetime('now') WHERE id = ?", (mid,))
|
||||
sent += 1
|
||||
return sent
|
||||
|
||||
|
||||
class Consumer:
|
||||
"""幂等消费者:按 message_id 去重(真实项目里用数据库唯一约束记录已处理的消息)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.processed: set[int] = set()
|
||||
self.points: dict[int, int] = {}
|
||||
|
||||
def handle(self, msg: dict) -> None:
|
||||
if msg["message_id"] in self.processed:
|
||||
return
|
||||
self.points[msg["user_id"]] = self.points.get(msg["user_id"], 0) + msg["amount"] // 100
|
||||
self.processed.add(msg["message_id"])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db = sqlite3.connect(":memory:")
|
||||
init(db)
|
||||
create_order(db, user_id=1, amount=19900)
|
||||
try:
|
||||
create_order(db, user_id=2, amount=500, fail_after_insert=True)
|
||||
except RuntimeError:
|
||||
pass
|
||||
create_order(db, user_id=1, amount=10000)
|
||||
assert db.execute("SELECT count(*) FROM orders").fetchone()[0] == 2
|
||||
assert db.execute("SELECT count(*) FROM outbox").fetchone()[0] == 2 # 回滚的订单没有留下消息
|
||||
|
||||
broker = FakeBroker(fail_times=1)
|
||||
assert relay(db, broker) == 0 # 第一次 broker 不可用
|
||||
assert relay(db, broker) == 2 # 重试成功
|
||||
assert relay(db, broker) == 0 # 没有待发送的了
|
||||
|
||||
consumer = Consumer()
|
||||
for m in broker.sent + broker.sent[:1]: # 模拟第一条消息被重复投递
|
||||
consumer.handle(m)
|
||||
assert consumer.points == {1: 199 + 100}, consumer.points
|
||||
print("发件箱:订单和消息同事务写入;投递失败可重试;消费者幂等,重复消息不会重复加积分")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user