"""AI 写作会话定期清理。 按 ``config.yaml`` 的 ``ai_writing.cleanup`` 段做联动清理: 1. 找出 MySQL ``ai_writing_sessions`` 表里 ``updated_at < now - retention_days`` 的会话 id 列表(分批,避免大事务) 2. 对每个 id 调用 LangGraph checkpointer 的 ``adelete_thread()``,删掉 SQLite checkpointer 里对应的 thread 状态 3. 再把 MySQL 行批量删掉 整个流程「先删 checkpointer 再删 DB」是有意为之 —— 即便清理被中途中断, 也只会出现「checkpointer 删了但 DB 没删」的孤儿,下一轮清理会发现这些 DB 行仍然过期,重新尝试清理(checkpointer 删一个不存在的 thread 是 idempotent)。 反过来如果先删 DB,万一 checkpointer 删失败,文件里就永久留着不可达的状态。 多 worker 下用文件锁做 leader election,只有抢到锁的 worker 才会真正跑清理。 """ from __future__ import annotations import asyncio import logging import sys from datetime import UTC, datetime, timedelta from pathlib import Path from croniter import croniter logger = logging.getLogger(__name__) # ── 文件锁 ──────────────────────────────────────────────────────────────────── def _acquire_lock(lock_path: Path): """跨平台非阻塞独占文件锁,多 worker 下做 leader election。 成功 → 返回持有锁的文件句柄(调用方需保持其存活直到主动释放); 抢不到 / 打开失败 → 返回 None。进程退出时 OS 自动释放。 """ try: lock_path.parent.mkdir(parents=True, exist_ok=True) handle = open(lock_path, "a+") except OSError: logger.exception("ai_writing_cleanup: 打开锁文件失败 path=%s", lock_path) return None try: if sys.platform == "win32": import msvcrt handle.seek(0) msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) else: import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: handle.close() return None return handle def _release_lock(handle) -> None: try: if sys.platform == "win32": import msvcrt handle.seek(0) msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) else: import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_UN) except OSError: pass finally: try: handle.close() except OSError: pass # ── 清理执行 ────────────────────────────────────────────────────────────────── async def _delete_checkpoint_thread(checkpointer, thread_id: str) -> None: """删 checkpointer 里 thread_id 对应的全部状态。失败只记日志,不抛。""" if checkpointer is None: return # LangGraph 异步删法首选 adelete_thread;老版本只有同步 delete_thread。 delete_async = getattr(checkpointer, "adelete_thread", None) if delete_async is not None: try: await delete_async(thread_id) return except Exception: logger.warning( "ai_writing_cleanup: adelete_thread failed thread_id=%s", thread_id, exc_info=True ) return delete_sync = getattr(checkpointer, "delete_thread", None) if delete_sync is not None: try: # 同步方法跑在线程池里,避免堵 event loop await asyncio.to_thread(delete_sync, thread_id) except Exception: logger.warning( "ai_writing_cleanup: delete_thread failed thread_id=%s", thread_id, exc_info=True ) def _get_active_session_ids() -> set[str]: """返回内存里**当前正在跑**的 session_id 集合。 这些会话的写作流程还活着 —— 任何清理都不能动它们,否则用户「正在写」 的稿子会被秒删,体验灾难。延迟到下一轮清理(24 小时后)再处理也比误 删强。 """ try: # 延迟 import 避免循环依赖:cleanup 模块在 lifespan 启动时被 import, # routers/ai_writing 也被 include_router 主动 import,两者拓扑无环但 # 还是放函数里更稳。 from app.gateway.routers.ai_writing import _event_queues return set(_event_queues.keys()) except Exception: # 兜底:拿不到就当作没有活动会话(最差就是清掉一个刚跑完的会话, # 比误删正在写的要好得多 —— 用户至少能从 DB 历史里看到结果) logger.warning("ai_writing_cleanup: 读取活动会话列表失败", exc_info=True) return set() async def run_cleanup_once( *, retention_days: int, only_finished: bool = False, batch_size: int = 500, ) -> dict: """执行一轮清理。返回统计信息。 **安全保证**:清理永远跳过内存里仍在跑的会话(``_event_queues`` 里有 对应 key 的)—— 这条规则即使在 admin 手动触发清理时也强制生效,防止 把用户「正在写」的稿子误删。 幂等 + 可重入 —— 多次调用对结果只会越扫越干净。即便清理途中被中断,下次 再跑会重新把 DB 里仍然过期的行扫到(checkpointer 删一个已不存在的 thread 是 no-op)。 """ from deerflow.agents.ai_writing.graph import get_ai_writing_checkpointer from deerflow.persistence.ai_writing_sessions import AIWritingSessionRepository from deerflow.persistence.engine import get_session_factory sf = get_session_factory() if sf is None: logger.info("ai_writing_cleanup: persistence backend=memory,跳过清理") return {"deleted_db": 0, "deleted_checkpoints": 0, "skipped_active": 0} repo = AIWritingSessionRepository(sf) checkpointer = get_ai_writing_checkpointer() active_ids = _get_active_session_ids() cutoff = datetime.now(UTC) - timedelta(days=retention_days) logger.info( "ai_writing_cleanup: 开始清理 cutoff=%s only_finished=%s batch_size=%d active=%d", cutoff.isoformat(), only_finished, batch_size, len(active_ids), ) total_db = 0 total_cp = 0 total_skipped = 0 while True: ids = await repo.list_older_than(cutoff, only_finished=only_finished, limit=batch_size) if not ids: break # 关键保护:跳过任何仍在内存里跑的会话 ids_to_delete = [sid for sid in ids if sid not in active_ids] skipped = len(ids) - len(ids_to_delete) if skipped: logger.warning( "ai_writing_cleanup: 跳过 %d 个仍在跑的会话: %s", skipped, [sid for sid in ids if sid in active_ids], ) total_skipped += skipped if not ids_to_delete: # 本批全是活动会话 —— 但 list_older_than 是按 updated_at 升序的, # 下一批拿出来的更旧,所以这种情况只可能整个 cutoff 区间都被活动 # 会话占满,不用继续往下扫,break break # 先删 checkpointer,再删 DB —— 保证「DB 删了 → checkpointer 一定也清干净」 for sid in ids_to_delete: await _delete_checkpoint_thread(checkpointer, sid) total_cp += 1 deleted = await repo.delete_by_ids(ids_to_delete) total_db += deleted logger.info( "ai_writing_cleanup: 本批删 %d 行(DB),%d 个 thread(checkpointer),跳过 %d", deleted, len(ids_to_delete), skipped, ) # 整批比 batch_size 小 → 没有更多了 if len(ids) < batch_size: break logger.info( "ai_writing_cleanup: 清理完成 total_db=%d total_checkpoints=%d skipped_active=%d", total_db, total_cp, total_skipped, ) return { "deleted_db": total_db, "deleted_checkpoints": total_cp, "skipped_active": total_skipped, } # ── 调度循环 ────────────────────────────────────────────────────────────────── def _next_cron_delay_seconds(cron_expr: str, base: datetime | None = None) -> float: """根据 cron 表达式算到下一次触发的秒数。 用 ``datetime.now(UTC)`` 做基准。croniter 接受 naive 时间,这里转成 naive UTC 喂进去;当然如果用户希望按本地时区跑(凌晨 3 点),需要在外层调整 —— 我们用本地时区基准,等价于 cron 在本地时区跑。 """ base = base or datetime.now() nxt = croniter(cron_expr, base).get_next(datetime) return max(1.0, (nxt - base).total_seconds()) async def cleanup_scheduler_loop() -> None: """常驻后台任务:按 cron 调度跑清理。 多 worker 部署下文件锁做 leader election,没拿到锁的 worker 直接退出。 被 cancel 时 CancelledError 不会被 except Exception 吞掉,能干净退出。 """ from deerflow.config.app_config import get_app_config from deerflow.config.runtime_paths import runtime_home try: cfg = get_app_config() except Exception: logger.exception("ai_writing_cleanup: 加载 app config 失败,调度器退出") return raw = getattr(cfg, "ai_writing", None) or {} cleanup_cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {} if not cleanup_cfg.get("enabled", False): logger.info("ai_writing_cleanup: cleanup.enabled=false,调度器不启动") return cron_expr = str(cleanup_cfg.get("cron", "0 3 * * *")) retention_days = int(cleanup_cfg.get("retention_days", 7)) only_finished = bool(cleanup_cfg.get("delete_only_finished", False)) batch_size = int(cleanup_cfg.get("batch_size", 500)) # 提前校验 cron 表达式,配错就早 fail 早暴露 try: croniter(cron_expr, datetime.now()) except Exception: logger.exception("ai_writing_cleanup: 非法 cron 表达式 %r,调度器退出", cron_expr) return lock_handle = _acquire_lock(runtime_home() / ".ai_writing_cleanup.lock") if lock_handle is None: logger.info("ai_writing_cleanup: 另一个 worker 已持有调度锁,本 worker 跳过") return logger.info( "ai_writing_cleanup: 已启动 cron=%r retention_days=%d only_finished=%s batch_size=%d", cron_expr, retention_days, only_finished, batch_size, ) try: while True: delay = _next_cron_delay_seconds(cron_expr) logger.info("ai_writing_cleanup: 下一次清理 %.0f 秒后", delay) await asyncio.sleep(delay) try: await run_cleanup_once( retention_days=retention_days, only_finished=only_finished, batch_size=batch_size, ) except Exception: logger.exception("ai_writing_cleanup: 本轮清理失败(不影响下一轮)") finally: _release_lock(lock_handle)