Merge remote-tracking branch 'origin/master'

This commit is contained in:
吉鑫升 committed 2026-05-20 17:06:06 +08:00
commit 313290b22c
21 files changed
+6764 -2745

No files matched your search

@@ -71,6 +71,12 @@ logger = logging.getLogger(__name__)
# firing signals into a worker that is stuck waiting for shutdown cleanup.
_SHUTDOWN_HOOK_TIMEOUT_SECONDS = 5.0
# AI 写作 in-flight task 收尾的最大等待时间。一个 graph 片段通常在几秒内
# 跑到下一个 interrupt 或结束,但单段草稿生成走完整个 LLM 可能耗时较久,
# 给 30 秒兜底;超时则 cancel 强制结束,已落盘的 checkpoint 仍可用于重启
# 后恢复,丢的只是最后一段 in-flight 但未 checkpoint 的进度。
_AI_WRITING_DRAIN_TIMEOUT_SECONDS = 30.0
async def _ensure_admin_user(app: FastAPI) -> None:
"""Startup hook: handle first boot and migrate orphan threads otherwise.
@@ -362,10 +368,68 @@ async def _curator_scheduler_loop() -> None:
_release_singleton_lock(lock_handle)
def _warn_if_multi_worker() -> None:
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
AI 写作 + 主 agent 流式响应都依赖**进程内**的内存状态字典
(``_event_queues``、``_session_pending_pause`` 等)。多 worker 下,
用户的 /start /stream /resume 会被 OS 随机分配到不同 worker,
per-worker 的内存字典互相看不见 → SSE 事件丢失、会话「凭空消失」、
resume 走不通。
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
"""
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
# / uvicorn 都认)。
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
logger.error(
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
web_concurrency,
)
return
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
if not sys.platform.startswith("linux"):
return
try:
from pathlib import Path
peers = 0
for entry in Path("/proc").iterdir():
if not entry.name.isdigit():
continue
try:
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
except OSError:
continue
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
peers += 1
if peers > 1:
logger.error(
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
"启用 sticky session。",
peers,
)
except Exception:
# 检测本身失败不算事 —— 这只是个提醒
pass
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
_warn_if_multi_worker()
# Load config and check necessary environment variables at startup
try:
app.state.config = get_app_config()
@@ -403,6 +467,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill curator scheduler failed to start (non-fatal)")
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
try:
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop())
except Exception:
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
@@ -440,6 +515,44 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Curator scheduler shutdown error")
# 停止 AI 写作清理调度器
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
if cleanup_task is not None:
cleanup_task.cancel()
try:
await cleanup_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("AI writing cleanup scheduler shutdown error")
# 等待所有 in-flight 写作 graph task 跑到下一个 interrupt / 结束。
# 这是「写作流程能完整继续」的关键 —— 如果直接关 checkpointer
# connection,正在跑 astream 的 task 会在写 checkpoint 时抛错,
# 已经生成但还没落盘的进度全丢。
try:
from app.gateway.routers.ai_writing import wait_for_inflight_graph_tasks
await wait_for_inflight_graph_tasks(timeout=_AI_WRITING_DRAIN_TIMEOUT_SECONDS)
except Exception:
logger.exception("Wait for in-flight AI writing tasks failed (non-fatal)")
# 关闭 AI 写作 graph 持有的 checkpointer 连接(aiosqlite Connection / pg pool)
try:
from deerflow.agents.ai_writing.graph import shutdown_ai_writing_graph
await asyncio.wait_for(
shutdown_ai_writing_graph(),
timeout=_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
)
except TimeoutError:
logger.warning(
"AI writing checkpointer shutdown exceeded %.1fs; proceeding with worker exit.",
_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
)
except Exception:
logger.exception("Failed to shutdown AI writing checkpointer")
try:
scheduler_service = getattr(app.state, "scheduled_task_service", None)
if scheduler_service is not None: