容错处理

This commit is contained in:
董宏凯 committed 2026-05-20 12:51:04 +08:00
1 parent f44e2ccf52
commit f358cfb7c7
8 files changed
+988 -21

No files matched your search

@@ -70,6 +70,28 @@ class RenameSessionRequest(BaseModel):
title: str
class AdminCleanupRequest(BaseModel):
"""手动清理接口的可选覆盖参数。
全部留空 → 用 config.yaml 的 ai_writing.cleanup 段默认值;显式传值
→ 覆盖配置(一般用于运维想验证一个临时阈值,例如清掉「最近 1 小时
以前的」会话来模拟整轮调度)。
"""
retention_days: Optional[int] = Field(default=None, ge=0, description="保留多少天以内的会话;0 = 不保留")
only_finished: Optional[bool] = Field(default=None, description="True 时只清 done/error 状态")
batch_size: Optional[int] = Field(default=None, ge=1, le=10000, description="单批最多删多少行")
class AdminCleanupResponse(BaseModel):
deleted_db: int
deleted_checkpoints: int
# 因「仍在跑」被保护掉的会话数。永远跳过 _event_queues 里活跃的会话,
# 即使 retention_days=0 也不会秒杀用户正在写的稿。
skipped_active: int = 0
elapsed_seconds: float
params: dict
class SaveTranscriptRequest(BaseModel):
# 前端组装的完整对话时间线(progressEvents + completedInterventions)
transcript: dict
@@ -153,11 +175,40 @@ _session_pending_pause: dict[str, tuple] = {}
# lost session don't each rebuild a different queue object.
_recovery_lock = asyncio.Lock()
# 在飞的 graph 推进任务(每次 /start 和 /resume 都会塞一个进来)。
# Lifespan shutdown 时会等这些 task 跑到下一个 interrupt / 结束,再关闭
# checkpointer 连接 —— 否则中途关 connection,task 内部的 sqlite 写入会抛
# 错,导致 LLM 已经生成但还没落 checkpoint 的进度丢失。
_inflight_graph_tasks: set[asyncio.Task] = set()
# ── 工具函数 ──────────────────────────────────────────────────────────────────
def _sse(event: str, data: dict) -> str:
return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
"""组装一帧 SSE。
JSON 序列化对 progress_events 来说理论上不会失败(节点里塞的都是 str/dict
/list/int 这类基础类型),但一旦塞进了 datetime / bytes / 自定义对象,
``json.dumps`` 会抛 TypeError → SSE 生成器爆掉 → 客户端再也收不到任何
事件,包括关键的 await_user 和 done。
这里用 ``default=str`` 兜底任何「带 __str__」的对象;如果连 ``str()``
都炸了,最后还兜一次手写一条 error 帧推出去 —— 写作流程「能继续」比
「漂亮地序列化」更重要。
"""
try:
payload = json.dumps(data, ensure_ascii=False, default=str)
except Exception as exc:
logger.exception("ai_writing: SSE payload 序列化失败 event=%s", event)
payload = json.dumps(
{
"session_id": data.get("session_id") if isinstance(data, dict) else None,
"message": f"事件序列化失败(已降级): {type(exc).__name__}",
},
ensure_ascii=False,
)
event = "notice" # 不要硬塞回原 event,万一是 error 会污染语义
return f"event: {event}\ndata: {payload}\n\n"
def _ai_writing_config() -> dict:
@@ -193,14 +244,81 @@ def _truncate_title(text: str, max_len: int = 50) -> str:
return text[:max_len] + ("…" if len(text) > max_len else "")
def _spawn_graph_task(coro) -> asyncio.Task:
"""启一个 graph 推进 task 并登记到 _inflight_graph_tasks。
用 ``add_done_callback`` 在完成时自动反登记 —— 不管 task 正常结束、抛
异常、还是被 cancel,都会清理;这样 _inflight_graph_tasks 不会泄漏。
"""
task = asyncio.create_task(coro)
_inflight_graph_tasks.add(task)
task.add_done_callback(_inflight_graph_tasks.discard)
return task
async def wait_for_inflight_graph_tasks(timeout: float) -> tuple[int, int]:
"""Lifespan shutdown 入口:阻塞等待所有 in-flight graph task 收尾。
每个 ``_stream_graph_segment`` 跑到下一个 ``interrupt()`` 或 ``END`` 就
会自然返回;通常只需要等几秒到几十秒。超时后剩下的会被 ``cancel()``,
但 cancel 完仍要等它们 ack 退出(带极短的 grace),否则下一步关
checkpointer 时还会撞上活的协程。
Returns:
(drained, cancelled): 自然结束的数量、被 cancel 强制结束的数量
"""
if not _inflight_graph_tasks:
return 0, 0
pending = set(_inflight_graph_tasks)
initial = len(pending)
logger.info("ai_writing: 等待 %d 个 in-flight graph task 收尾(timeout=%.1fs)", initial, timeout)
done, still_running = await asyncio.wait(pending, timeout=timeout)
drained = len(done)
cancelled = 0
if still_running:
logger.warning(
"ai_writing: 仍有 %d 个 task 超时未结束,强制 cancel", len(still_running)
)
for t in still_running:
t.cancel()
cancelled += 1
# 给 cancel 一点 ack 时间,避免下游关连接时撞活协程
await asyncio.wait(still_running, timeout=2.0)
logger.info("ai_writing: in-flight 收尾完成 drained=%d cancelled=%d", drained, cancelled)
return drained, cancelled
# ── 路由 ──────────────────────────────────────────────────────────────────────
@router.post("/start", response_model=StartWritingResponse)
async def start_writing(body: StartWritingRequest, request: Request):
"""启动写作流程,返回 session_id。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
"""启动写作流程,返回 session_id。
**失败兜底契约**:本接口除非 Pydantic 校验 body 失败(422),其他任何
后端组件(DB / checkpointer / 文章类型仓库 / 用户解析)失败都不会导致
/start 返回 5xx。哪怕兜底走的是「极简内存模式」也要先把 session_id
交给前端,让它能起 SSE 接到 error 事件,避免 UI 卡死在 loading 状态。
"""
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
from app.gateway.deps import get_ai_writing_session_store, get_article_type_store, get_current_user
# get_ai_writing_graph() 内部已对 sqlite/postgres 失败做了 InMemorySaver
# 降级,理论上永不抛。但 build_ai_writing_graph 拼图结构也可能因下游
# 节点 import 失败抛错 —— 这种情况 graph 完全不可用,必须早 fail,前端
# 弹错处理。
try:
graph = await get_ai_writing_graph()
except Exception as e:
logger.exception("ai_writing: graph 构建彻底失败,无法启动")
raise HTTPException(
status_code=503,
detail=f"AI 写作服务暂不可用: {type(e).__name__}",
) from e
session_id = str(uuid.uuid4())
queue: asyncio.Queue = asyncio.Queue()
_event_queues[session_id] = queue
@@ -258,7 +376,7 @@ async def start_writing(body: StartWritingRequest, request: Request):
}
config = _make_config(session_id, body.model_name, valid_article_types=valid_article_types)
asyncio.create_task(_stream_graph_segment(session_id, initial_state, config, AI_WRITING_GRAPH))
_spawn_graph_task(_stream_graph_segment(session_id, initial_state, config, graph))
return StartWritingResponse(session_id=session_id)
@@ -430,9 +548,11 @@ async def stream_progress(session_id: str):
@router.post("/{session_id}/resume")
async def resume_writing(session_id: str, body: ResumeRequest, request: Request):
"""用户提交干预决策,恢复暂停的流程。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
from app.gateway.deps import get_current_user
graph = await get_ai_writing_graph()
if session_id not in _event_queues:
# 内存会话注册表丢失(进程内清理后重连等)。先尝试从 checkpointer 恢复,
# 让用户能在干预点继续,而不是直接 404 中断整个写作流程。
@@ -482,8 +602,8 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
# Clear pending pause so SSE reconnect won't re-inject the stale await_user event
_session_pending_pause.pop(session_id, None)
asyncio.create_task(
_stream_graph_segment(session_id, Command(resume=intervention), config, AI_WRITING_GRAPH)
_spawn_graph_task(
_stream_graph_segment(session_id, Command(resume=intervention), config, graph)
)
return {"message": "已提交,流程恢复中"}
@@ -491,15 +611,112 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
@router.get("/{session_id}/state")
async def get_state(session_id: str):
"""查询当前流程状态快照。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
graph = await get_ai_writing_graph()
config = _make_config(session_id)
snapshot = await AI_WRITING_GRAPH.aget_state(config)
snapshot = await graph.aget_state(config)
if not snapshot or not snapshot.values:
raise HTTPException(status_code=404, detail="Session not found")
return snapshot.values
# ── 管理接口 ──────────────────────────────────────────────────────────────────
async def _require_admin(request: Request) -> None:
"""放行 admin / auth-disabled,其余 403。
与 scheduled_tasks._require_admin 同套语义:auth_disabled 模式下
get_optional_user_from_request 返回 None,此时放行(开发环境一致体验)。
"""
from app.gateway.deps import get_optional_user_from_request
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="AI 写作清理管理接口仅限管理员")
def _resolve_cleanup_params(override: AdminCleanupRequest) -> dict:
"""合并 config.yaml 默认值 + 请求显式覆盖。
显式传 0 / False 也算覆盖(用 ``is not None`` 判定),方便测「保留 0 天 =
全删」这种极端场景。
"""
from deerflow.config import get_app_config
raw = getattr(get_app_config(), "ai_writing", None) or {}
cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
return {
"retention_days": (
override.retention_days if override.retention_days is not None
else int(cfg.get("retention_days", 7))
),
"only_finished": (
override.only_finished if override.only_finished is not None
else bool(cfg.get("delete_only_finished", False))
),
"batch_size": (
override.batch_size if override.batch_size is not None
else int(cfg.get("batch_size", 500))
),
}
@router.post("/admin/cleanup", response_model=AdminCleanupResponse)
async def admin_cleanup(
body: AdminCleanupRequest,
request: Request,
) -> AdminCleanupResponse:
"""**仅管理员** 手动触发一次会话清理(不等凌晨 3 点定时调度)。
用途:
- 验证 cleanup 配置改动是否生效
- 内网紧急清理(磁盘满 / MySQL 行数过多)
- 调试:传 ``retention_days: 0`` 一刷拉清空,方便重新跑回归
所有参数留空 → 沿用 config.yaml 的 ``ai_writing.cleanup`` 段默认。显式
传值会覆盖配置但**不修改 config.yaml**(一次性参数,下次定时调度还是用
配置文件的值)。
"""
import time
from app.gateway.ai_writing_cleanup import run_cleanup_once
await _require_admin(request)
params = _resolve_cleanup_params(body)
started = time.monotonic()
try:
result = await run_cleanup_once(**params)
except Exception as e:
logger.exception("ai_writing admin cleanup failed: params=%s", params)
# 抛 500 是合理的 —— 管理员触发清理失败需要看到失败原因,跟「业务
# 接口被失败拖累」是两回事
raise HTTPException(
status_code=500,
detail=f"清理执行失败: {type(e).__name__}: {e}",
) from e
elapsed = time.monotonic() - started
logger.info(
"ai_writing admin cleanup done: params=%s deleted_db=%d deleted_cp=%d elapsed=%.2fs",
params,
result["deleted_db"],
result["deleted_checkpoints"],
elapsed,
)
return AdminCleanupResponse(
deleted_db=result["deleted_db"],
deleted_checkpoints=result["deleted_checkpoints"],
skipped_active=result.get("skipped_active", 0),
elapsed_seconds=round(elapsed, 3),
params=params,
)
# ── 内部辅助 ──────────────────────────────────────────────────────────────────
async def _stream_graph_segment(
@@ -650,13 +867,14 @@ def _event_type(evt: dict) -> str:
async def _load_graph_snapshot(session_id: str):
"""读取 AI 写作 Graph 在该 session 上的状态快照(失败/无状态返回 None)。
注意:Graph 用的是内存 checkpointer(MemorySaver),所以这里只能在
**进程存活**期间恢复;进程/容器重启后快照同样会丢失。
Graph 的 checkpointer 现在按 config.yaml 持久化到磁盘(默认 SQLite),
所以容器重启后只要数据卷还在,仍然能恢复出来。
"""
try:
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
snapshot = await AI_WRITING_GRAPH.aget_state(_make_config(session_id))
graph = await get_ai_writing_graph()
snapshot = await graph.aget_state(_make_config(session_id))
except Exception:
logger.exception("ai_writing: aget_state failed during recovery: session=%s", session_id)
return None