Merge remote-tracking branch 'origin/master'
This commit is contained in:
commit
313290b22c
21 files changed
+6764
-2745
No files matched your search
@@ -71,6 +71,12 @@ logger = logging.getLogger(__name__)
|
||||
# firing signals into a worker that is stuck waiting for shutdown cleanup.
|
||||
_SHUTDOWN_HOOK_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
# AI 写作 in-flight task 收尾的最大等待时间。一个 graph 片段通常在几秒内
|
||||
# 跑到下一个 interrupt 或结束,但单段草稿生成走完整个 LLM 可能耗时较久,
|
||||
# 给 30 秒兜底;超时则 cancel 强制结束,已落盘的 checkpoint 仍可用于重启
|
||||
# 后恢复,丢的只是最后一段 in-flight 但未 checkpoint 的进度。
|
||||
_AI_WRITING_DRAIN_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
"""Startup hook: handle first boot and migrate orphan threads otherwise.
|
||||
@@ -362,10 +368,68 @@ async def _curator_scheduler_loop() -> None:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
|
||||
AI 写作 + 主 agent 流式响应都依赖**进程内**的内存状态字典
|
||||
(``_event_queues``、``_session_pending_pause`` 等)。多 worker 下,
|
||||
用户的 /start /stream /resume 会被 OS 随机分配到不同 worker,
|
||||
per-worker 的内存字典互相看不见 → SSE 事件丢失、会话「凭空消失」、
|
||||
resume 走不通。
|
||||
|
||||
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
|
||||
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
|
||||
"""
|
||||
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
|
||||
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
|
||||
# / uvicorn 都认)。
|
||||
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
|
||||
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
|
||||
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
|
||||
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
|
||||
web_concurrency,
|
||||
)
|
||||
return
|
||||
|
||||
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
peers = 0
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
|
||||
peers += 1
|
||||
if peers > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
|
||||
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
|
||||
"启用 sticky session。",
|
||||
peers,
|
||||
)
|
||||
except Exception:
|
||||
# 检测本身失败不算事 —— 这只是个提醒
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
|
||||
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
|
||||
_warn_if_multi_worker()
|
||||
|
||||
# Load config and check necessary environment variables at startup
|
||||
try:
|
||||
app.state.config = get_app_config()
|
||||
@@ -403,6 +467,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -440,6 +515,44 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
# 停止 AI 写作清理调度器
|
||||
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
|
||||
if cleanup_task is not None:
|
||||
cleanup_task.cancel()
|
||||
try:
|
||||
await cleanup_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler shutdown error")
|
||||
|
||||
# 等待所有 in-flight 写作 graph task 跑到下一个 interrupt / 结束。
|
||||
# 这是「写作流程能完整继续」的关键 —— 如果直接关 checkpointer
|
||||
# connection,正在跑 astream 的 task 会在写 checkpoint 时抛错,
|
||||
# 已经生成但还没落盘的进度全丢。
|
||||
try:
|
||||
from app.gateway.routers.ai_writing import wait_for_inflight_graph_tasks
|
||||
|
||||
await wait_for_inflight_graph_tasks(timeout=_AI_WRITING_DRAIN_TIMEOUT_SECONDS)
|
||||
except Exception:
|
||||
logger.exception("Wait for in-flight AI writing tasks failed (non-fatal)")
|
||||
|
||||
# 关闭 AI 写作 graph 持有的 checkpointer 连接(aiosqlite Connection / pg pool)
|
||||
try:
|
||||
from deerflow.agents.ai_writing.graph import shutdown_ai_writing_graph
|
||||
|
||||
await asyncio.wait_for(
|
||||
shutdown_ai_writing_graph(),
|
||||
timeout=_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
"AI writing checkpointer shutdown exceeded %.1fs; proceeding with worker exit.",
|
||||
_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to shutdown AI writing checkpointer")
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
if scheduler_service is not None:
|
||||
|
||||
Reference in new issue
Block a user