容错处理

This commit is contained in:
董宏凯 committed 2026-05-20 12:51:04 +08:00
1 parent f44e2ccf52
commit f358cfb7c7
8 files changed
+988 -21

No files matched your search

@@ -0,0 +1,300 @@
"""AI 写作会话定期清理。
按 ``config.yaml`` 的 ``ai_writing.cleanup`` 段做联动清理:
1. 找出 MySQL ``ai_writing_sessions`` 表里 ``updated_at < now - retention_days``
的会话 id 列表(分批,避免大事务)
2. 对每个 id 调用 LangGraph checkpointer 的 ``adelete_thread()``,删掉
SQLite checkpointer 里对应的 thread 状态
3. 再把 MySQL 行批量删掉
整个流程「先删 checkpointer 再删 DB」是有意为之 —— 即便清理被中途中断,
也只会出现「checkpointer 删了但 DB 没删」的孤儿,下一轮清理会发现这些
DB 行仍然过期,重新尝试清理(checkpointer 删一个不存在的 thread 是 idempotent)。
反过来如果先删 DB,万一 checkpointer 删失败,文件里就永久留着不可达的状态。
多 worker 下用文件锁做 leader election,只有抢到锁的 worker 才会真正跑清理。
"""
from __future__ import annotations
import asyncio
import logging
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
from croniter import croniter
logger = logging.getLogger(__name__)
# ── 文件锁 ────────────────────────────────────────────────────────────────────
def _acquire_lock(lock_path: Path):
"""跨平台非阻塞独占文件锁,多 worker 下做 leader election。
成功 → 返回持有锁的文件句柄(调用方需保持其存活直到主动释放);
抢不到 / 打开失败 → 返回 None。进程退出时 OS 自动释放。
"""
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = open(lock_path, "a+")
except OSError:
logger.exception("ai_writing_cleanup: 打开锁文件失败 path=%s", lock_path)
return None
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
return None
return handle
def _release_lock(handle) -> None:
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError:
pass
finally:
try:
handle.close()
except OSError:
pass
# ── 清理执行 ──────────────────────────────────────────────────────────────────
async def _delete_checkpoint_thread(checkpointer, thread_id: str) -> None:
"""删 checkpointer 里 thread_id 对应的全部状态。失败只记日志,不抛。"""
if checkpointer is None:
return
# LangGraph 异步删法首选 adelete_thread;老版本只有同步 delete_thread。
delete_async = getattr(checkpointer, "adelete_thread", None)
if delete_async is not None:
try:
await delete_async(thread_id)
return
except Exception:
logger.warning(
"ai_writing_cleanup: adelete_thread failed thread_id=%s", thread_id, exc_info=True
)
return
delete_sync = getattr(checkpointer, "delete_thread", None)
if delete_sync is not None:
try:
# 同步方法跑在线程池里,避免堵 event loop
await asyncio.to_thread(delete_sync, thread_id)
except Exception:
logger.warning(
"ai_writing_cleanup: delete_thread failed thread_id=%s", thread_id, exc_info=True
)
def _get_active_session_ids() -> set[str]:
"""返回内存里**当前正在跑**的 session_id 集合。
这些会话的写作流程还活着 —— 任何清理都不能动它们,否则用户「正在写」
的稿子会被秒删,体验灾难。延迟到下一轮清理(24 小时后)再处理也比误
删强。
"""
try:
# 延迟 import 避免循环依赖:cleanup 模块在 lifespan 启动时被 import,
# routers/ai_writing 也被 include_router 主动 import,两者拓扑无环但
# 还是放函数里更稳。
from app.gateway.routers.ai_writing import _event_queues
return set(_event_queues.keys())
except Exception:
# 兜底:拿不到就当作没有活动会话(最差就是清掉一个刚跑完的会话,
# 比误删正在写的要好得多 —— 用户至少能从 DB 历史里看到结果)
logger.warning("ai_writing_cleanup: 读取活动会话列表失败", exc_info=True)
return set()
async def run_cleanup_once(
*,
retention_days: int,
only_finished: bool = False,
batch_size: int = 500,
) -> dict:
"""执行一轮清理。返回统计信息。
**安全保证**:清理永远跳过内存里仍在跑的会话(``_event_queues`` 里有
对应 key 的)—— 这条规则即使在 admin 手动触发清理时也强制生效,防止
把用户「正在写」的稿子误删。
幂等 + 可重入 —— 多次调用对结果只会越扫越干净。即便清理途中被中断,下次
再跑会重新把 DB 里仍然过期的行扫到(checkpointer 删一个已不存在的 thread
是 no-op)。
"""
from deerflow.agents.ai_writing.graph import get_ai_writing_checkpointer
from deerflow.persistence.ai_writing_sessions import AIWritingSessionRepository
from deerflow.persistence.engine import get_session_factory
sf = get_session_factory()
if sf is None:
logger.info("ai_writing_cleanup: persistence backend=memory,跳过清理")
return {"deleted_db": 0, "deleted_checkpoints": 0, "skipped_active": 0}
repo = AIWritingSessionRepository(sf)
checkpointer = get_ai_writing_checkpointer()
active_ids = _get_active_session_ids()
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
logger.info(
"ai_writing_cleanup: 开始清理 cutoff=%s only_finished=%s batch_size=%d active=%d",
cutoff.isoformat(),
only_finished,
batch_size,
len(active_ids),
)
total_db = 0
total_cp = 0
total_skipped = 0
while True:
ids = await repo.list_older_than(cutoff, only_finished=only_finished, limit=batch_size)
if not ids:
break
# 关键保护:跳过任何仍在内存里跑的会话
ids_to_delete = [sid for sid in ids if sid not in active_ids]
skipped = len(ids) - len(ids_to_delete)
if skipped:
logger.warning(
"ai_writing_cleanup: 跳过 %d 个仍在跑的会话: %s",
skipped,
[sid for sid in ids if sid in active_ids],
)
total_skipped += skipped
if not ids_to_delete:
# 本批全是活动会话 —— 但 list_older_than 是按 updated_at 升序的,
# 下一批拿出来的更旧,所以这种情况只可能整个 cutoff 区间都被活动
# 会话占满,不用继续往下扫,break
break
# 先删 checkpointer,再删 DB —— 保证「DB 删了 → checkpointer 一定也清干净」
for sid in ids_to_delete:
await _delete_checkpoint_thread(checkpointer, sid)
total_cp += 1
deleted = await repo.delete_by_ids(ids_to_delete)
total_db += deleted
logger.info(
"ai_writing_cleanup: 本批删 %d 行(DB),%d 个 thread(checkpointer),跳过 %d",
deleted,
len(ids_to_delete),
skipped,
)
# 整批比 batch_size 小 → 没有更多了
if len(ids) < batch_size:
break
logger.info(
"ai_writing_cleanup: 清理完成 total_db=%d total_checkpoints=%d skipped_active=%d",
total_db,
total_cp,
total_skipped,
)
return {
"deleted_db": total_db,
"deleted_checkpoints": total_cp,
"skipped_active": total_skipped,
}
# ── 调度循环 ──────────────────────────────────────────────────────────────────
def _next_cron_delay_seconds(cron_expr: str, base: datetime | None = None) -> float:
"""根据 cron 表达式算到下一次触发的秒数。
用 ``datetime.now(UTC)`` 做基准。croniter 接受 naive 时间,这里转成 naive
UTC 喂进去;当然如果用户希望按本地时区跑(凌晨 3 点),需要在外层调整 ——
我们用本地时区基准,等价于 cron 在本地时区跑。
"""
base = base or datetime.now()
nxt = croniter(cron_expr, base).get_next(datetime)
return max(1.0, (nxt - base).total_seconds())
async def cleanup_scheduler_loop() -> None:
"""常驻后台任务:按 cron 调度跑清理。
多 worker 部署下文件锁做 leader election,没拿到锁的 worker 直接退出。
被 cancel 时 CancelledError 不会被 except Exception 吞掉,能干净退出。
"""
from deerflow.config.app_config import get_app_config
from deerflow.config.runtime_paths import runtime_home
try:
cfg = get_app_config()
except Exception:
logger.exception("ai_writing_cleanup: 加载 app config 失败,调度器退出")
return
raw = getattr(cfg, "ai_writing", None) or {}
cleanup_cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
if not cleanup_cfg.get("enabled", False):
logger.info("ai_writing_cleanup: cleanup.enabled=false,调度器不启动")
return
cron_expr = str(cleanup_cfg.get("cron", "0 3 * * *"))
retention_days = int(cleanup_cfg.get("retention_days", 7))
only_finished = bool(cleanup_cfg.get("delete_only_finished", False))
batch_size = int(cleanup_cfg.get("batch_size", 500))
# 提前校验 cron 表达式,配错就早 fail 早暴露
try:
croniter(cron_expr, datetime.now())
except Exception:
logger.exception("ai_writing_cleanup: 非法 cron 表达式 %r,调度器退出", cron_expr)
return
lock_handle = _acquire_lock(runtime_home() / ".ai_writing_cleanup.lock")
if lock_handle is None:
logger.info("ai_writing_cleanup: 另一个 worker 已持有调度锁,本 worker 跳过")
return
logger.info(
"ai_writing_cleanup: 已启动 cron=%r retention_days=%d only_finished=%s batch_size=%d",
cron_expr,
retention_days,
only_finished,
batch_size,
)
try:
while True:
delay = _next_cron_delay_seconds(cron_expr)
logger.info("ai_writing_cleanup: 下一次清理 %.0f 秒后", delay)
await asyncio.sleep(delay)
try:
await run_cleanup_once(
retention_days=retention_days,
only_finished=only_finished,
batch_size=batch_size,
)
except Exception:
logger.exception("ai_writing_cleanup: 本轮清理失败(不影响下一轮)")
finally:
_release_lock(lock_handle)