报错兜底
This commit is contained in:
1 parent
226439ef36
commit
f44e2ccf52
10 files changed
+3010
-2726
No files matched your search
@@ -149,6 +149,10 @@ _session_article_types: dict[str, list[dict]] = {}
|
||||
# so that a new SSE connection (e.g., after user clicks "继续") can re-inject it.
|
||||
_session_pending_pause: dict[str, tuple] = {}
|
||||
|
||||
# Serializes session recovery so that a concurrent /stream + /resume on the same
|
||||
# lost session don't each rebuild a different queue object.
|
||||
_recovery_lock = asyncio.Lock()
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -317,16 +321,26 @@ async def delete_session(session_id: str, request: Request):
|
||||
|
||||
@router.put("/sessions/{session_id}/transcript")
|
||||
async def save_transcript(session_id: str, body: SaveTranscriptRequest, request: Request):
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。"""
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。
|
||||
|
||||
时间线持久化属于「锦上添花」功能:即便保存失败(如内网部署缺少 transcript
|
||||
列、或 MySQL TEXT 列超长),也绝不能阻断写作流程。任何异常都吞掉并返回
|
||||
success=False,由前端忽略即可。
|
||||
"""
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_current_user
|
||||
|
||||
repo = get_ai_writing_session_store(request)
|
||||
if repo is None:
|
||||
return {"success": False}
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
return {"success": False, "reason": "store_unavailable"}
|
||||
try:
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save transcript: session=%s", session_id)
|
||||
return {"success": False, "reason": "save_failed"}
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
# 会话不存在或不属于当前用户:同样不抛 500,返回 success=False 即可
|
||||
return {"success": False, "reason": "session_not_found"}
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@@ -420,7 +434,17 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
|
||||
from app.gateway.deps import get_current_user
|
||||
|
||||
if session_id not in _event_queues:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
# 内存会话注册表丢失(进程内清理后重连等)。先尝试从 checkpointer 恢复,
|
||||
# 让用户能在干预点继续,而不是直接 404 中断整个写作流程。
|
||||
recovery = await _recover_session(session_id, request)
|
||||
if recovery != "interrupted":
|
||||
logger.warning(
|
||||
"ai_writing: resume on unrecoverable session=%s (recovery=%s)", session_id, recovery
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="会话状态已丢失或流程已结束,无法继续,请重新发起写作。",
|
||||
)
|
||||
|
||||
config = _make_config(session_id)
|
||||
intervention = {
|
||||
@@ -623,6 +647,104 @@ def _event_type(evt: dict) -> str:
|
||||
return t
|
||||
|
||||
|
||||
async def _load_graph_snapshot(session_id: str):
|
||||
"""读取 AI 写作 Graph 在该 session 上的状态快照(失败/无状态返回 None)。
|
||||
|
||||
注意:Graph 用的是内存 checkpointer(MemorySaver),所以这里只能在
|
||||
**进程存活**期间恢复;进程/容器重启后快照同样会丢失。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
|
||||
snapshot = await AI_WRITING_GRAPH.aget_state(_make_config(session_id))
|
||||
except Exception:
|
||||
logger.exception("ai_writing: aget_state failed during recovery: session=%s", session_id)
|
||||
return None
|
||||
if not snapshot or not getattr(snapshot, "values", None):
|
||||
return None
|
||||
return snapshot
|
||||
|
||||
|
||||
def _extract_interrupt(snapshot) -> dict | None:
|
||||
"""从快照里取出当前挂起的 interrupt 负载(即 await_user 的数据)。"""
|
||||
for task in getattr(snapshot, "tasks", None) or ():
|
||||
for intr in getattr(task, "interrupts", None) or ():
|
||||
value = getattr(intr, "value", None)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
async def _recover_session(session_id: str, request: Request | None = None) -> str:
|
||||
"""内存会话注册表丢失时,尝试从 checkpointer 重建会话。
|
||||
|
||||
返回:
|
||||
- "interrupted" —— 图正停在某个干预点,已重建队列并预置 await_user 事件,可继续 resume
|
||||
- "finished" —— 图已完成,已重建队列并预置 done 事件(+草稿)
|
||||
- "unrecoverable" —— 无可恢复状态(进程/容器重启过,或 session 从未存在)
|
||||
|
||||
幂等:内存里已有队列时直接当作 "interrupted" 返回。整段在 `_recovery_lock`
|
||||
下执行,避免并发的 /stream 与 /resume 各自重建出不同的队列对象。
|
||||
"""
|
||||
async with _recovery_lock:
|
||||
if session_id in _event_queues:
|
||||
return "interrupted"
|
||||
|
||||
snapshot = await _load_graph_snapshot(session_id)
|
||||
if snapshot is None:
|
||||
return "unrecoverable"
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
_event_queues[session_id] = queue
|
||||
_session_interventions.setdefault(session_id, [])
|
||||
_session_model_names.setdefault(session_id, "")
|
||||
_session_article_types.setdefault(session_id, [])
|
||||
|
||||
# 尽量恢复 DB 仓库引用,使后续 resume 仍能持久化草稿/审核结果
|
||||
if request is not None and session_id not in _session_repos:
|
||||
try:
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_current_user
|
||||
|
||||
repo = get_ai_writing_session_store(request)
|
||||
if repo is not None:
|
||||
_session_repos[session_id] = repo
|
||||
_session_user_ids[session_id] = await get_current_user(request)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to restore repo during recovery: session=%s", session_id)
|
||||
|
||||
interrupt_val = _extract_interrupt(snapshot)
|
||||
if interrupt_val is not None:
|
||||
event = ("await_user", {**interrupt_val, "session_id": session_id})
|
||||
await queue.put(event)
|
||||
_session_pending_pause[session_id] = event
|
||||
logger.info("ai_writing: recovered interrupted session from checkpointer: session=%s", session_id)
|
||||
return "interrupted"
|
||||
|
||||
values = snapshot.values or {}
|
||||
if not getattr(snapshot, "next", None) or values.get("status") == "done":
|
||||
draft = values.get("current_draft") or {}
|
||||
if draft.get("full_markdown"):
|
||||
await queue.put(("draft_ready", {
|
||||
"session_id": session_id,
|
||||
"title": draft.get("title", ""),
|
||||
"markdown": draft.get("full_markdown", ""),
|
||||
"revision_count": draft.get("revision_count", 0),
|
||||
}))
|
||||
await queue.put(("done", {"session_id": session_id, "message": "写作已完成"}))
|
||||
await queue.put(None) # 哨兵:让 SSE 生成器读完即关闭
|
||||
logger.info("ai_writing: recovered finished session from checkpointer: session=%s", session_id)
|
||||
return "finished"
|
||||
|
||||
# 既不在干预点、也未结束:上次运行多半被中断在某个节点,无法续推
|
||||
await queue.put(("error", {
|
||||
"session_id": session_id,
|
||||
"message": "写作流程已中断,请重新发起写作。",
|
||||
}))
|
||||
await queue.put(None)
|
||||
logger.warning("ai_writing: session checkpoint stuck mid-flow, not resumable: session=%s", session_id)
|
||||
return "finished"
|
||||
|
||||
|
||||
async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
|
||||
"""从队列消费事件,推送为 SSE 帧。"""
|
||||
# 等待队列创建(最多 5 秒)
|
||||
@@ -631,10 +753,24 @@ async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
yield _sse("error", {"message": "Session not found"})
|
||||
return
|
||||
# 内存里没有这个会话队列。不要直接报「Session not found」——
|
||||
# 先尝试从 checkpointer 恢复(图可能仍停在某个干预点 / 已完成)。
|
||||
recovery = await _recover_session(session_id)
|
||||
if recovery == "unrecoverable":
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
yield _sse("error", {
|
||||
"session_id": session_id,
|
||||
"message": "会话状态已丢失(服务可能已重启),请重新发起写作。",
|
||||
})
|
||||
return
|
||||
# 恢复成功:_event_queues[session_id] 已重建并预置事件,继续走正常消费逻辑。
|
||||
|
||||
queue = _event_queues[session_id]
|
||||
queue = _event_queues.get(session_id)
|
||||
if queue is None:
|
||||
# 极端兜底:恢复后又被并发清理
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
yield _sse("error", {"session_id": session_id, "message": "会话状态已丢失,请重新发起写作。"})
|
||||
return
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
|
||||
# Re-inject the saved await_user event if the queue is empty and a pending pause exists.
|
||||
|
||||
Reference in new issue
Block a user