容错处理
This commit is contained in:
1 parent
f44e2ccf52
commit
f358cfb7c7
8 files changed
+988
-21
No files matched your search
@@ -70,6 +70,28 @@ class RenameSessionRequest(BaseModel):
|
||||
title: str
|
||||
|
||||
|
||||
class AdminCleanupRequest(BaseModel):
|
||||
"""手动清理接口的可选覆盖参数。
|
||||
|
||||
全部留空 → 用 config.yaml 的 ai_writing.cleanup 段默认值;显式传值
|
||||
→ 覆盖配置(一般用于运维想验证一个临时阈值,例如清掉「最近 1 小时
|
||||
以前的」会话来模拟整轮调度)。
|
||||
"""
|
||||
retention_days: Optional[int] = Field(default=None, ge=0, description="保留多少天以内的会话;0 = 不保留")
|
||||
only_finished: Optional[bool] = Field(default=None, description="True 时只清 done/error 状态")
|
||||
batch_size: Optional[int] = Field(default=None, ge=1, le=10000, description="单批最多删多少行")
|
||||
|
||||
|
||||
class AdminCleanupResponse(BaseModel):
|
||||
deleted_db: int
|
||||
deleted_checkpoints: int
|
||||
# 因「仍在跑」被保护掉的会话数。永远跳过 _event_queues 里活跃的会话,
|
||||
# 即使 retention_days=0 也不会秒杀用户正在写的稿。
|
||||
skipped_active: int = 0
|
||||
elapsed_seconds: float
|
||||
params: dict
|
||||
|
||||
|
||||
class SaveTranscriptRequest(BaseModel):
|
||||
# 前端组装的完整对话时间线(progressEvents + completedInterventions)
|
||||
transcript: dict
|
||||
@@ -153,11 +175,40 @@ _session_pending_pause: dict[str, tuple] = {}
|
||||
# lost session don't each rebuild a different queue object.
|
||||
_recovery_lock = asyncio.Lock()
|
||||
|
||||
# 在飞的 graph 推进任务(每次 /start 和 /resume 都会塞一个进来)。
|
||||
# Lifespan shutdown 时会等这些 task 跑到下一个 interrupt / 结束,再关闭
|
||||
# checkpointer 连接 —— 否则中途关 connection,task 内部的 sqlite 写入会抛
|
||||
# 错,导致 LLM 已经生成但还没落 checkpoint 的进度丢失。
|
||||
_inflight_graph_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _sse(event: str, data: dict) -> str:
|
||||
return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
"""组装一帧 SSE。
|
||||
|
||||
JSON 序列化对 progress_events 来说理论上不会失败(节点里塞的都是 str/dict
|
||||
/list/int 这类基础类型),但一旦塞进了 datetime / bytes / 自定义对象,
|
||||
``json.dumps`` 会抛 TypeError → SSE 生成器爆掉 → 客户端再也收不到任何
|
||||
事件,包括关键的 await_user 和 done。
|
||||
|
||||
这里用 ``default=str`` 兜底任何「带 __str__」的对象;如果连 ``str()``
|
||||
都炸了,最后还兜一次手写一条 error 帧推出去 —— 写作流程「能继续」比
|
||||
「漂亮地序列化」更重要。
|
||||
"""
|
||||
try:
|
||||
payload = json.dumps(data, ensure_ascii=False, default=str)
|
||||
except Exception as exc:
|
||||
logger.exception("ai_writing: SSE payload 序列化失败 event=%s", event)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"session_id": data.get("session_id") if isinstance(data, dict) else None,
|
||||
"message": f"事件序列化失败(已降级): {type(exc).__name__}",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
event = "notice" # 不要硬塞回原 event,万一是 error 会污染语义
|
||||
return f"event: {event}\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
def _ai_writing_config() -> dict:
|
||||
@@ -193,14 +244,81 @@ def _truncate_title(text: str, max_len: int = 50) -> str:
|
||||
return text[:max_len] + ("…" if len(text) > max_len else "")
|
||||
|
||||
|
||||
def _spawn_graph_task(coro) -> asyncio.Task:
|
||||
"""启一个 graph 推进 task 并登记到 _inflight_graph_tasks。
|
||||
|
||||
用 ``add_done_callback`` 在完成时自动反登记 —— 不管 task 正常结束、抛
|
||||
异常、还是被 cancel,都会清理;这样 _inflight_graph_tasks 不会泄漏。
|
||||
"""
|
||||
task = asyncio.create_task(coro)
|
||||
_inflight_graph_tasks.add(task)
|
||||
task.add_done_callback(_inflight_graph_tasks.discard)
|
||||
return task
|
||||
|
||||
|
||||
async def wait_for_inflight_graph_tasks(timeout: float) -> tuple[int, int]:
|
||||
"""Lifespan shutdown 入口:阻塞等待所有 in-flight graph task 收尾。
|
||||
|
||||
每个 ``_stream_graph_segment`` 跑到下一个 ``interrupt()`` 或 ``END`` 就
|
||||
会自然返回;通常只需要等几秒到几十秒。超时后剩下的会被 ``cancel()``,
|
||||
但 cancel 完仍要等它们 ack 退出(带极短的 grace),否则下一步关
|
||||
checkpointer 时还会撞上活的协程。
|
||||
|
||||
Returns:
|
||||
(drained, cancelled): 自然结束的数量、被 cancel 强制结束的数量
|
||||
"""
|
||||
if not _inflight_graph_tasks:
|
||||
return 0, 0
|
||||
|
||||
pending = set(_inflight_graph_tasks)
|
||||
initial = len(pending)
|
||||
logger.info("ai_writing: 等待 %d 个 in-flight graph task 收尾(timeout=%.1fs)", initial, timeout)
|
||||
|
||||
done, still_running = await asyncio.wait(pending, timeout=timeout)
|
||||
drained = len(done)
|
||||
|
||||
cancelled = 0
|
||||
if still_running:
|
||||
logger.warning(
|
||||
"ai_writing: 仍有 %d 个 task 超时未结束,强制 cancel", len(still_running)
|
||||
)
|
||||
for t in still_running:
|
||||
t.cancel()
|
||||
cancelled += 1
|
||||
# 给 cancel 一点 ack 时间,避免下游关连接时撞活协程
|
||||
await asyncio.wait(still_running, timeout=2.0)
|
||||
|
||||
logger.info("ai_writing: in-flight 收尾完成 drained=%d cancelled=%d", drained, cancelled)
|
||||
return drained, cancelled
|
||||
|
||||
|
||||
# ── 路由 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/start", response_model=StartWritingResponse)
|
||||
async def start_writing(body: StartWritingRequest, request: Request):
|
||||
"""启动写作流程,返回 session_id。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
"""启动写作流程,返回 session_id。
|
||||
|
||||
**失败兜底契约**:本接口除非 Pydantic 校验 body 失败(422),其他任何
|
||||
后端组件(DB / checkpointer / 文章类型仓库 / 用户解析)失败都不会导致
|
||||
/start 返回 5xx。哪怕兜底走的是「极简内存模式」也要先把 session_id
|
||||
交给前端,让它能起 SSE 接到 error 事件,避免 UI 卡死在 loading 状态。
|
||||
"""
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_article_type_store, get_current_user
|
||||
|
||||
# get_ai_writing_graph() 内部已对 sqlite/postgres 失败做了 InMemorySaver
|
||||
# 降级,理论上永不抛。但 build_ai_writing_graph 拼图结构也可能因下游
|
||||
# 节点 import 失败抛错 —— 这种情况 graph 完全不可用,必须早 fail,前端
|
||||
# 弹错处理。
|
||||
try:
|
||||
graph = await get_ai_writing_graph()
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing: graph 构建彻底失败,无法启动")
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=f"AI 写作服务暂不可用: {type(e).__name__}",
|
||||
) from e
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
_event_queues[session_id] = queue
|
||||
@@ -258,7 +376,7 @@ async def start_writing(body: StartWritingRequest, request: Request):
|
||||
}
|
||||
config = _make_config(session_id, body.model_name, valid_article_types=valid_article_types)
|
||||
|
||||
asyncio.create_task(_stream_graph_segment(session_id, initial_state, config, AI_WRITING_GRAPH))
|
||||
_spawn_graph_task(_stream_graph_segment(session_id, initial_state, config, graph))
|
||||
return StartWritingResponse(session_id=session_id)
|
||||
|
||||
|
||||
@@ -430,9 +548,11 @@ async def stream_progress(session_id: str):
|
||||
@router.post("/{session_id}/resume")
|
||||
async def resume_writing(session_id: str, body: ResumeRequest, request: Request):
|
||||
"""用户提交干预决策,恢复暂停的流程。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
from app.gateway.deps import get_current_user
|
||||
|
||||
graph = await get_ai_writing_graph()
|
||||
|
||||
if session_id not in _event_queues:
|
||||
# 内存会话注册表丢失(进程内清理后重连等)。先尝试从 checkpointer 恢复,
|
||||
# 让用户能在干预点继续,而不是直接 404 中断整个写作流程。
|
||||
@@ -482,8 +602,8 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
|
||||
# Clear pending pause so SSE reconnect won't re-inject the stale await_user event
|
||||
_session_pending_pause.pop(session_id, None)
|
||||
|
||||
asyncio.create_task(
|
||||
_stream_graph_segment(session_id, Command(resume=intervention), config, AI_WRITING_GRAPH)
|
||||
_spawn_graph_task(
|
||||
_stream_graph_segment(session_id, Command(resume=intervention), config, graph)
|
||||
)
|
||||
return {"message": "已提交,流程恢复中"}
|
||||
|
||||
@@ -491,15 +611,112 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
|
||||
@router.get("/{session_id}/state")
|
||||
async def get_state(session_id: str):
|
||||
"""查询当前流程状态快照。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
|
||||
graph = await get_ai_writing_graph()
|
||||
config = _make_config(session_id)
|
||||
snapshot = await AI_WRITING_GRAPH.aget_state(config)
|
||||
snapshot = await graph.aget_state(config)
|
||||
if not snapshot or not snapshot.values:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
return snapshot.values
|
||||
|
||||
|
||||
# ── 管理接口 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
"""放行 admin / auth-disabled,其余 403。
|
||||
|
||||
与 scheduled_tasks._require_admin 同套语义:auth_disabled 模式下
|
||||
get_optional_user_from_request 返回 None,此时放行(开发环境一致体验)。
|
||||
"""
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="AI 写作清理管理接口仅限管理员")
|
||||
|
||||
|
||||
def _resolve_cleanup_params(override: AdminCleanupRequest) -> dict:
|
||||
"""合并 config.yaml 默认值 + 请求显式覆盖。
|
||||
|
||||
显式传 0 / False 也算覆盖(用 ``is not None`` 判定),方便测「保留 0 天 =
|
||||
全删」这种极端场景。
|
||||
"""
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
raw = getattr(get_app_config(), "ai_writing", None) or {}
|
||||
cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
|
||||
|
||||
return {
|
||||
"retention_days": (
|
||||
override.retention_days if override.retention_days is not None
|
||||
else int(cfg.get("retention_days", 7))
|
||||
),
|
||||
"only_finished": (
|
||||
override.only_finished if override.only_finished is not None
|
||||
else bool(cfg.get("delete_only_finished", False))
|
||||
),
|
||||
"batch_size": (
|
||||
override.batch_size if override.batch_size is not None
|
||||
else int(cfg.get("batch_size", 500))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/admin/cleanup", response_model=AdminCleanupResponse)
|
||||
async def admin_cleanup(
|
||||
body: AdminCleanupRequest,
|
||||
request: Request,
|
||||
) -> AdminCleanupResponse:
|
||||
"""**仅管理员** 手动触发一次会话清理(不等凌晨 3 点定时调度)。
|
||||
|
||||
用途:
|
||||
- 验证 cleanup 配置改动是否生效
|
||||
- 内网紧急清理(磁盘满 / MySQL 行数过多)
|
||||
- 调试:传 ``retention_days: 0`` 一刷拉清空,方便重新跑回归
|
||||
|
||||
所有参数留空 → 沿用 config.yaml 的 ``ai_writing.cleanup`` 段默认。显式
|
||||
传值会覆盖配置但**不修改 config.yaml**(一次性参数,下次定时调度还是用
|
||||
配置文件的值)。
|
||||
"""
|
||||
import time
|
||||
|
||||
from app.gateway.ai_writing_cleanup import run_cleanup_once
|
||||
|
||||
await _require_admin(request)
|
||||
params = _resolve_cleanup_params(body)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = await run_cleanup_once(**params)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing admin cleanup failed: params=%s", params)
|
||||
# 抛 500 是合理的 —— 管理员触发清理失败需要看到失败原因,跟「业务
|
||||
# 接口被失败拖累」是两回事
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"清理执行失败: {type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
logger.info(
|
||||
"ai_writing admin cleanup done: params=%s deleted_db=%d deleted_cp=%d elapsed=%.2fs",
|
||||
params,
|
||||
result["deleted_db"],
|
||||
result["deleted_checkpoints"],
|
||||
elapsed,
|
||||
)
|
||||
return AdminCleanupResponse(
|
||||
deleted_db=result["deleted_db"],
|
||||
deleted_checkpoints=result["deleted_checkpoints"],
|
||||
skipped_active=result.get("skipped_active", 0),
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
params=params,
|
||||
)
|
||||
|
||||
|
||||
# ── 内部辅助 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _stream_graph_segment(
|
||||
@@ -650,13 +867,14 @@ def _event_type(evt: dict) -> str:
|
||||
async def _load_graph_snapshot(session_id: str):
|
||||
"""读取 AI 写作 Graph 在该 session 上的状态快照(失败/无状态返回 None)。
|
||||
|
||||
注意:Graph 用的是内存 checkpointer(MemorySaver),所以这里只能在
|
||||
**进程存活**期间恢复;进程/容器重启后快照同样会丢失。
|
||||
Graph 的 checkpointer 现在按 config.yaml 持久化到磁盘(默认 SQLite),
|
||||
所以容器重启后只要数据卷还在,仍然能恢复出来。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
|
||||
snapshot = await AI_WRITING_GRAPH.aget_state(_make_config(session_id))
|
||||
graph = await get_ai_writing_graph()
|
||||
snapshot = await graph.aget_state(_make_config(session_id))
|
||||
except Exception:
|
||||
logger.exception("ai_writing: aget_state failed during recovery: session=%s", session_id)
|
||||
return None
|
||||
|
||||
Reference in new issue
Block a user