Merge remote-tracking branch 'origin/master'
This commit is contained in:
commit
313290b22c
21 files changed
+6764
-2745
No files matched your search
@@ -0,0 +1,300 @@
|
||||
"""AI 写作会话定期清理。
|
||||
|
||||
按 ``config.yaml`` 的 ``ai_writing.cleanup`` 段做联动清理:
|
||||
|
||||
1. 找出 MySQL ``ai_writing_sessions`` 表里 ``updated_at < now - retention_days``
|
||||
的会话 id 列表(分批,避免大事务)
|
||||
2. 对每个 id 调用 LangGraph checkpointer 的 ``adelete_thread()``,删掉
|
||||
SQLite checkpointer 里对应的 thread 状态
|
||||
3. 再把 MySQL 行批量删掉
|
||||
|
||||
整个流程「先删 checkpointer 再删 DB」是有意为之 —— 即便清理被中途中断,
|
||||
也只会出现「checkpointer 删了但 DB 没删」的孤儿,下一轮清理会发现这些
|
||||
DB 行仍然过期,重新尝试清理(checkpointer 删一个不存在的 thread 是 idempotent)。
|
||||
反过来如果先删 DB,万一 checkpointer 删失败,文件里就永久留着不可达的状态。
|
||||
|
||||
多 worker 下用文件锁做 leader election,只有抢到锁的 worker 才会真正跑清理。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── 文件锁 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _acquire_lock(lock_path: Path):
|
||||
"""跨平台非阻塞独占文件锁,多 worker 下做 leader election。
|
||||
|
||||
成功 → 返回持有锁的文件句柄(调用方需保持其存活直到主动释放);
|
||||
抢不到 / 打开失败 → 返回 None。进程退出时 OS 自动释放。
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("ai_writing_cleanup: 打开锁文件失败 path=%s", lock_path)
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
return handle
|
||||
|
||||
|
||||
def _release_lock(handle) -> None:
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
handle.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ── 清理执行 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _delete_checkpoint_thread(checkpointer, thread_id: str) -> None:
|
||||
"""删 checkpointer 里 thread_id 对应的全部状态。失败只记日志,不抛。"""
|
||||
if checkpointer is None:
|
||||
return
|
||||
# LangGraph 异步删法首选 adelete_thread;老版本只有同步 delete_thread。
|
||||
delete_async = getattr(checkpointer, "adelete_thread", None)
|
||||
if delete_async is not None:
|
||||
try:
|
||||
await delete_async(thread_id)
|
||||
return
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: adelete_thread failed thread_id=%s", thread_id, exc_info=True
|
||||
)
|
||||
return
|
||||
delete_sync = getattr(checkpointer, "delete_thread", None)
|
||||
if delete_sync is not None:
|
||||
try:
|
||||
# 同步方法跑在线程池里,避免堵 event loop
|
||||
await asyncio.to_thread(delete_sync, thread_id)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: delete_thread failed thread_id=%s", thread_id, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
def _get_active_session_ids() -> set[str]:
|
||||
"""返回内存里**当前正在跑**的 session_id 集合。
|
||||
|
||||
这些会话的写作流程还活着 —— 任何清理都不能动它们,否则用户「正在写」
|
||||
的稿子会被秒删,体验灾难。延迟到下一轮清理(24 小时后)再处理也比误
|
||||
删强。
|
||||
"""
|
||||
try:
|
||||
# 延迟 import 避免循环依赖:cleanup 模块在 lifespan 启动时被 import,
|
||||
# routers/ai_writing 也被 include_router 主动 import,两者拓扑无环但
|
||||
# 还是放函数里更稳。
|
||||
from app.gateway.routers.ai_writing import _event_queues
|
||||
|
||||
return set(_event_queues.keys())
|
||||
except Exception:
|
||||
# 兜底:拿不到就当作没有活动会话(最差就是清掉一个刚跑完的会话,
|
||||
# 比误删正在写的要好得多 —— 用户至少能从 DB 历史里看到结果)
|
||||
logger.warning("ai_writing_cleanup: 读取活动会话列表失败", exc_info=True)
|
||||
return set()
|
||||
|
||||
|
||||
async def run_cleanup_once(
|
||||
*,
|
||||
retention_days: int,
|
||||
only_finished: bool = False,
|
||||
batch_size: int = 500,
|
||||
) -> dict:
|
||||
"""执行一轮清理。返回统计信息。
|
||||
|
||||
**安全保证**:清理永远跳过内存里仍在跑的会话(``_event_queues`` 里有
|
||||
对应 key 的)—— 这条规则即使在 admin 手动触发清理时也强制生效,防止
|
||||
把用户「正在写」的稿子误删。
|
||||
|
||||
幂等 + 可重入 —— 多次调用对结果只会越扫越干净。即便清理途中被中断,下次
|
||||
再跑会重新把 DB 里仍然过期的行扫到(checkpointer 删一个已不存在的 thread
|
||||
是 no-op)。
|
||||
"""
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_checkpointer
|
||||
from deerflow.persistence.ai_writing_sessions import AIWritingSessionRepository
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
logger.info("ai_writing_cleanup: persistence backend=memory,跳过清理")
|
||||
return {"deleted_db": 0, "deleted_checkpoints": 0, "skipped_active": 0}
|
||||
|
||||
repo = AIWritingSessionRepository(sf)
|
||||
checkpointer = get_ai_writing_checkpointer()
|
||||
active_ids = _get_active_session_ids()
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 开始清理 cutoff=%s only_finished=%s batch_size=%d active=%d",
|
||||
cutoff.isoformat(),
|
||||
only_finished,
|
||||
batch_size,
|
||||
len(active_ids),
|
||||
)
|
||||
|
||||
total_db = 0
|
||||
total_cp = 0
|
||||
total_skipped = 0
|
||||
while True:
|
||||
ids = await repo.list_older_than(cutoff, only_finished=only_finished, limit=batch_size)
|
||||
if not ids:
|
||||
break
|
||||
|
||||
# 关键保护:跳过任何仍在内存里跑的会话
|
||||
ids_to_delete = [sid for sid in ids if sid not in active_ids]
|
||||
skipped = len(ids) - len(ids_to_delete)
|
||||
if skipped:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: 跳过 %d 个仍在跑的会话: %s",
|
||||
skipped,
|
||||
[sid for sid in ids if sid in active_ids],
|
||||
)
|
||||
total_skipped += skipped
|
||||
|
||||
if not ids_to_delete:
|
||||
# 本批全是活动会话 —— 但 list_older_than 是按 updated_at 升序的,
|
||||
# 下一批拿出来的更旧,所以这种情况只可能整个 cutoff 区间都被活动
|
||||
# 会话占满,不用继续往下扫,break
|
||||
break
|
||||
|
||||
# 先删 checkpointer,再删 DB —— 保证「DB 删了 → checkpointer 一定也清干净」
|
||||
for sid in ids_to_delete:
|
||||
await _delete_checkpoint_thread(checkpointer, sid)
|
||||
total_cp += 1
|
||||
|
||||
deleted = await repo.delete_by_ids(ids_to_delete)
|
||||
total_db += deleted
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 本批删 %d 行(DB),%d 个 thread(checkpointer),跳过 %d",
|
||||
deleted,
|
||||
len(ids_to_delete),
|
||||
skipped,
|
||||
)
|
||||
|
||||
# 整批比 batch_size 小 → 没有更多了
|
||||
if len(ids) < batch_size:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 清理完成 total_db=%d total_checkpoints=%d skipped_active=%d",
|
||||
total_db,
|
||||
total_cp,
|
||||
total_skipped,
|
||||
)
|
||||
return {
|
||||
"deleted_db": total_db,
|
||||
"deleted_checkpoints": total_cp,
|
||||
"skipped_active": total_skipped,
|
||||
}
|
||||
|
||||
|
||||
# ── 调度循环 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _next_cron_delay_seconds(cron_expr: str, base: datetime | None = None) -> float:
|
||||
"""根据 cron 表达式算到下一次触发的秒数。
|
||||
|
||||
用 ``datetime.now(UTC)`` 做基准。croniter 接受 naive 时间,这里转成 naive
|
||||
UTC 喂进去;当然如果用户希望按本地时区跑(凌晨 3 点),需要在外层调整 ——
|
||||
我们用本地时区基准,等价于 cron 在本地时区跑。
|
||||
"""
|
||||
base = base or datetime.now()
|
||||
nxt = croniter(cron_expr, base).get_next(datetime)
|
||||
return max(1.0, (nxt - base).total_seconds())
|
||||
|
||||
|
||||
async def cleanup_scheduler_loop() -> None:
|
||||
"""常驻后台任务:按 cron 调度跑清理。
|
||||
|
||||
多 worker 部署下文件锁做 leader election,没拿到锁的 worker 直接退出。
|
||||
被 cancel 时 CancelledError 不会被 except Exception 吞掉,能干净退出。
|
||||
"""
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
try:
|
||||
cfg = get_app_config()
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 加载 app config 失败,调度器退出")
|
||||
return
|
||||
|
||||
raw = getattr(cfg, "ai_writing", None) or {}
|
||||
cleanup_cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
|
||||
if not cleanup_cfg.get("enabled", False):
|
||||
logger.info("ai_writing_cleanup: cleanup.enabled=false,调度器不启动")
|
||||
return
|
||||
|
||||
cron_expr = str(cleanup_cfg.get("cron", "0 3 * * *"))
|
||||
retention_days = int(cleanup_cfg.get("retention_days", 7))
|
||||
only_finished = bool(cleanup_cfg.get("delete_only_finished", False))
|
||||
batch_size = int(cleanup_cfg.get("batch_size", 500))
|
||||
|
||||
# 提前校验 cron 表达式,配错就早 fail 早暴露
|
||||
try:
|
||||
croniter(cron_expr, datetime.now())
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 非法 cron 表达式 %r,调度器退出", cron_expr)
|
||||
return
|
||||
|
||||
lock_handle = _acquire_lock(runtime_home() / ".ai_writing_cleanup.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("ai_writing_cleanup: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 已启动 cron=%r retention_days=%d only_finished=%s batch_size=%d",
|
||||
cron_expr,
|
||||
retention_days,
|
||||
only_finished,
|
||||
batch_size,
|
||||
)
|
||||
try:
|
||||
while True:
|
||||
delay = _next_cron_delay_seconds(cron_expr)
|
||||
logger.info("ai_writing_cleanup: 下一次清理 %.0f 秒后", delay)
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
await run_cleanup_once(
|
||||
retention_days=retention_days,
|
||||
only_finished=only_finished,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 本轮清理失败(不影响下一轮)")
|
||||
finally:
|
||||
_release_lock(lock_handle)
|
||||
@@ -71,6 +71,12 @@ logger = logging.getLogger(__name__)
|
||||
# firing signals into a worker that is stuck waiting for shutdown cleanup.
|
||||
_SHUTDOWN_HOOK_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
# AI 写作 in-flight task 收尾的最大等待时间。一个 graph 片段通常在几秒内
|
||||
# 跑到下一个 interrupt 或结束,但单段草稿生成走完整个 LLM 可能耗时较久,
|
||||
# 给 30 秒兜底;超时则 cancel 强制结束,已落盘的 checkpoint 仍可用于重启
|
||||
# 后恢复,丢的只是最后一段 in-flight 但未 checkpoint 的进度。
|
||||
_AI_WRITING_DRAIN_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
"""Startup hook: handle first boot and migrate orphan threads otherwise.
|
||||
@@ -362,10 +368,68 @@ async def _curator_scheduler_loop() -> None:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
|
||||
AI 写作 + 主 agent 流式响应都依赖**进程内**的内存状态字典
|
||||
(``_event_queues``、``_session_pending_pause`` 等)。多 worker 下,
|
||||
用户的 /start /stream /resume 会被 OS 随机分配到不同 worker,
|
||||
per-worker 的内存字典互相看不见 → SSE 事件丢失、会话「凭空消失」、
|
||||
resume 走不通。
|
||||
|
||||
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
|
||||
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
|
||||
"""
|
||||
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
|
||||
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
|
||||
# / uvicorn 都认)。
|
||||
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
|
||||
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
|
||||
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
|
||||
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
|
||||
web_concurrency,
|
||||
)
|
||||
return
|
||||
|
||||
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
peers = 0
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
|
||||
peers += 1
|
||||
if peers > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
|
||||
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
|
||||
"启用 sticky session。",
|
||||
peers,
|
||||
)
|
||||
except Exception:
|
||||
# 检测本身失败不算事 —— 这只是个提醒
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
|
||||
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
|
||||
_warn_if_multi_worker()
|
||||
|
||||
# Load config and check necessary environment variables at startup
|
||||
try:
|
||||
app.state.config = get_app_config()
|
||||
@@ -403,6 +467,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -440,6 +515,44 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
# 停止 AI 写作清理调度器
|
||||
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
|
||||
if cleanup_task is not None:
|
||||
cleanup_task.cancel()
|
||||
try:
|
||||
await cleanup_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler shutdown error")
|
||||
|
||||
# 等待所有 in-flight 写作 graph task 跑到下一个 interrupt / 结束。
|
||||
# 这是「写作流程能完整继续」的关键 —— 如果直接关 checkpointer
|
||||
# connection,正在跑 astream 的 task 会在写 checkpoint 时抛错,
|
||||
# 已经生成但还没落盘的进度全丢。
|
||||
try:
|
||||
from app.gateway.routers.ai_writing import wait_for_inflight_graph_tasks
|
||||
|
||||
await wait_for_inflight_graph_tasks(timeout=_AI_WRITING_DRAIN_TIMEOUT_SECONDS)
|
||||
except Exception:
|
||||
logger.exception("Wait for in-flight AI writing tasks failed (non-fatal)")
|
||||
|
||||
# 关闭 AI 写作 graph 持有的 checkpointer 连接(aiosqlite Connection / pg pool)
|
||||
try:
|
||||
from deerflow.agents.ai_writing.graph import shutdown_ai_writing_graph
|
||||
|
||||
await asyncio.wait_for(
|
||||
shutdown_ai_writing_graph(),
|
||||
timeout=_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
"AI writing checkpointer shutdown exceeded %.1fs; proceeding with worker exit.",
|
||||
_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to shutdown AI writing checkpointer")
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
if scheduler_service is not None:
|
||||
|
||||
@@ -70,6 +70,28 @@ class RenameSessionRequest(BaseModel):
|
||||
title: str
|
||||
|
||||
|
||||
class AdminCleanupRequest(BaseModel):
|
||||
"""手动清理接口的可选覆盖参数。
|
||||
|
||||
全部留空 → 用 config.yaml 的 ai_writing.cleanup 段默认值;显式传值
|
||||
→ 覆盖配置(一般用于运维想验证一个临时阈值,例如清掉「最近 1 小时
|
||||
以前的」会话来模拟整轮调度)。
|
||||
"""
|
||||
retention_days: Optional[int] = Field(default=None, ge=0, description="保留多少天以内的会话;0 = 不保留")
|
||||
only_finished: Optional[bool] = Field(default=None, description="True 时只清 done/error 状态")
|
||||
batch_size: Optional[int] = Field(default=None, ge=1, le=10000, description="单批最多删多少行")
|
||||
|
||||
|
||||
class AdminCleanupResponse(BaseModel):
|
||||
deleted_db: int
|
||||
deleted_checkpoints: int
|
||||
# 因「仍在跑」被保护掉的会话数。永远跳过 _event_queues 里活跃的会话,
|
||||
# 即使 retention_days=0 也不会秒杀用户正在写的稿。
|
||||
skipped_active: int = 0
|
||||
elapsed_seconds: float
|
||||
params: dict
|
||||
|
||||
|
||||
class SaveTranscriptRequest(BaseModel):
|
||||
# 前端组装的完整对话时间线(progressEvents + completedInterventions)
|
||||
transcript: dict
|
||||
@@ -149,11 +171,44 @@ _session_article_types: dict[str, list[dict]] = {}
|
||||
# so that a new SSE connection (e.g., after user clicks "继续") can re-inject it.
|
||||
_session_pending_pause: dict[str, tuple] = {}
|
||||
|
||||
# Serializes session recovery so that a concurrent /stream + /resume on the same
|
||||
# lost session don't each rebuild a different queue object.
|
||||
_recovery_lock = asyncio.Lock()
|
||||
|
||||
# 在飞的 graph 推进任务(每次 /start 和 /resume 都会塞一个进来)。
|
||||
# Lifespan shutdown 时会等这些 task 跑到下一个 interrupt / 结束,再关闭
|
||||
# checkpointer 连接 —— 否则中途关 connection,task 内部的 sqlite 写入会抛
|
||||
# 错,导致 LLM 已经生成但还没落 checkpoint 的进度丢失。
|
||||
_inflight_graph_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _sse(event: str, data: dict) -> str:
|
||||
return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
"""组装一帧 SSE。
|
||||
|
||||
JSON 序列化对 progress_events 来说理论上不会失败(节点里塞的都是 str/dict
|
||||
/list/int 这类基础类型),但一旦塞进了 datetime / bytes / 自定义对象,
|
||||
``json.dumps`` 会抛 TypeError → SSE 生成器爆掉 → 客户端再也收不到任何
|
||||
事件,包括关键的 await_user 和 done。
|
||||
|
||||
这里用 ``default=str`` 兜底任何「带 __str__」的对象;如果连 ``str()``
|
||||
都炸了,最后还兜一次手写一条 error 帧推出去 —— 写作流程「能继续」比
|
||||
「漂亮地序列化」更重要。
|
||||
"""
|
||||
try:
|
||||
payload = json.dumps(data, ensure_ascii=False, default=str)
|
||||
except Exception as exc:
|
||||
logger.exception("ai_writing: SSE payload 序列化失败 event=%s", event)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"session_id": data.get("session_id") if isinstance(data, dict) else None,
|
||||
"message": f"事件序列化失败(已降级): {type(exc).__name__}",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
event = "notice" # 不要硬塞回原 event,万一是 error 会污染语义
|
||||
return f"event: {event}\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
def _ai_writing_config() -> dict:
|
||||
@@ -189,14 +244,81 @@ def _truncate_title(text: str, max_len: int = 50) -> str:
|
||||
return text[:max_len] + ("…" if len(text) > max_len else "")
|
||||
|
||||
|
||||
def _spawn_graph_task(coro) -> asyncio.Task:
|
||||
"""启一个 graph 推进 task 并登记到 _inflight_graph_tasks。
|
||||
|
||||
用 ``add_done_callback`` 在完成时自动反登记 —— 不管 task 正常结束、抛
|
||||
异常、还是被 cancel,都会清理;这样 _inflight_graph_tasks 不会泄漏。
|
||||
"""
|
||||
task = asyncio.create_task(coro)
|
||||
_inflight_graph_tasks.add(task)
|
||||
task.add_done_callback(_inflight_graph_tasks.discard)
|
||||
return task
|
||||
|
||||
|
||||
async def wait_for_inflight_graph_tasks(timeout: float) -> tuple[int, int]:
|
||||
"""Lifespan shutdown 入口:阻塞等待所有 in-flight graph task 收尾。
|
||||
|
||||
每个 ``_stream_graph_segment`` 跑到下一个 ``interrupt()`` 或 ``END`` 就
|
||||
会自然返回;通常只需要等几秒到几十秒。超时后剩下的会被 ``cancel()``,
|
||||
但 cancel 完仍要等它们 ack 退出(带极短的 grace),否则下一步关
|
||||
checkpointer 时还会撞上活的协程。
|
||||
|
||||
Returns:
|
||||
(drained, cancelled): 自然结束的数量、被 cancel 强制结束的数量
|
||||
"""
|
||||
if not _inflight_graph_tasks:
|
||||
return 0, 0
|
||||
|
||||
pending = set(_inflight_graph_tasks)
|
||||
initial = len(pending)
|
||||
logger.info("ai_writing: 等待 %d 个 in-flight graph task 收尾(timeout=%.1fs)", initial, timeout)
|
||||
|
||||
done, still_running = await asyncio.wait(pending, timeout=timeout)
|
||||
drained = len(done)
|
||||
|
||||
cancelled = 0
|
||||
if still_running:
|
||||
logger.warning(
|
||||
"ai_writing: 仍有 %d 个 task 超时未结束,强制 cancel", len(still_running)
|
||||
)
|
||||
for t in still_running:
|
||||
t.cancel()
|
||||
cancelled += 1
|
||||
# 给 cancel 一点 ack 时间,避免下游关连接时撞活协程
|
||||
await asyncio.wait(still_running, timeout=2.0)
|
||||
|
||||
logger.info("ai_writing: in-flight 收尾完成 drained=%d cancelled=%d", drained, cancelled)
|
||||
return drained, cancelled
|
||||
|
||||
|
||||
# ── 路由 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/start", response_model=StartWritingResponse)
|
||||
async def start_writing(body: StartWritingRequest, request: Request):
|
||||
"""启动写作流程,返回 session_id。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
"""启动写作流程,返回 session_id。
|
||||
|
||||
**失败兜底契约**:本接口除非 Pydantic 校验 body 失败(422),其他任何
|
||||
后端组件(DB / checkpointer / 文章类型仓库 / 用户解析)失败都不会导致
|
||||
/start 返回 5xx。哪怕兜底走的是「极简内存模式」也要先把 session_id
|
||||
交给前端,让它能起 SSE 接到 error 事件,避免 UI 卡死在 loading 状态。
|
||||
"""
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_article_type_store, get_current_user
|
||||
|
||||
# get_ai_writing_graph() 内部已对 sqlite/postgres 失败做了 InMemorySaver
|
||||
# 降级,理论上永不抛。但 build_ai_writing_graph 拼图结构也可能因下游
|
||||
# 节点 import 失败抛错 —— 这种情况 graph 完全不可用,必须早 fail,前端
|
||||
# 弹错处理。
|
||||
try:
|
||||
graph = await get_ai_writing_graph()
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing: graph 构建彻底失败,无法启动")
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=f"AI 写作服务暂不可用: {type(e).__name__}",
|
||||
) from e
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
_event_queues[session_id] = queue
|
||||
@@ -254,7 +376,7 @@ async def start_writing(body: StartWritingRequest, request: Request):
|
||||
}
|
||||
config = _make_config(session_id, body.model_name, valid_article_types=valid_article_types)
|
||||
|
||||
asyncio.create_task(_stream_graph_segment(session_id, initial_state, config, AI_WRITING_GRAPH))
|
||||
_spawn_graph_task(_stream_graph_segment(session_id, initial_state, config, graph))
|
||||
return StartWritingResponse(session_id=session_id)
|
||||
|
||||
|
||||
@@ -317,16 +439,26 @@ async def delete_session(session_id: str, request: Request):
|
||||
|
||||
@router.put("/sessions/{session_id}/transcript")
|
||||
async def save_transcript(session_id: str, body: SaveTranscriptRequest, request: Request):
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。"""
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。
|
||||
|
||||
时间线持久化属于「锦上添花」功能:即便保存失败(如内网部署缺少 transcript
|
||||
列、或 MySQL TEXT 列超长),也绝不能阻断写作流程。任何异常都吞掉并返回
|
||||
success=False,由前端忽略即可。
|
||||
"""
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_current_user
|
||||
|
||||
repo = get_ai_writing_session_store(request)
|
||||
if repo is None:
|
||||
return {"success": False}
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
return {"success": False, "reason": "store_unavailable"}
|
||||
try:
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save transcript: session=%s", session_id)
|
||||
return {"success": False, "reason": "save_failed"}
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
# 会话不存在或不属于当前用户:同样不抛 500,返回 success=False 即可
|
||||
return {"success": False, "reason": "session_not_found"}
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@@ -416,11 +548,23 @@ async def stream_progress(session_id: str):
|
||||
@router.post("/{session_id}/resume")
|
||||
async def resume_writing(session_id: str, body: ResumeRequest, request: Request):
|
||||
"""用户提交干预决策,恢复暂停的流程。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
from app.gateway.deps import get_current_user
|
||||
|
||||
graph = await get_ai_writing_graph()
|
||||
|
||||
if session_id not in _event_queues:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
# 内存会话注册表丢失(进程内清理后重连等)。先尝试从 checkpointer 恢复,
|
||||
# 让用户能在干预点继续,而不是直接 404 中断整个写作流程。
|
||||
recovery = await _recover_session(session_id, request)
|
||||
if recovery != "interrupted":
|
||||
logger.warning(
|
||||
"ai_writing: resume on unrecoverable session=%s (recovery=%s)", session_id, recovery
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="会话状态已丢失或流程已结束,无法继续,请重新发起写作。",
|
||||
)
|
||||
|
||||
config = _make_config(session_id)
|
||||
intervention = {
|
||||
@@ -458,8 +602,8 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
|
||||
# Clear pending pause so SSE reconnect won't re-inject the stale await_user event
|
||||
_session_pending_pause.pop(session_id, None)
|
||||
|
||||
asyncio.create_task(
|
||||
_stream_graph_segment(session_id, Command(resume=intervention), config, AI_WRITING_GRAPH)
|
||||
_spawn_graph_task(
|
||||
_stream_graph_segment(session_id, Command(resume=intervention), config, graph)
|
||||
)
|
||||
return {"message": "已提交,流程恢复中"}
|
||||
|
||||
@@ -467,15 +611,112 @@ async def resume_writing(session_id: str, body: ResumeRequest, request: Request)
|
||||
@router.get("/{session_id}/state")
|
||||
async def get_state(session_id: str):
|
||||
"""查询当前流程状态快照。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
|
||||
graph = await get_ai_writing_graph()
|
||||
config = _make_config(session_id)
|
||||
snapshot = await AI_WRITING_GRAPH.aget_state(config)
|
||||
snapshot = await graph.aget_state(config)
|
||||
if not snapshot or not snapshot.values:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
return snapshot.values
|
||||
|
||||
|
||||
# ── 管理接口 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
"""放行 admin / auth-disabled,其余 403。
|
||||
|
||||
与 scheduled_tasks._require_admin 同套语义:auth_disabled 模式下
|
||||
get_optional_user_from_request 返回 None,此时放行(开发环境一致体验)。
|
||||
"""
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="AI 写作清理管理接口仅限管理员")
|
||||
|
||||
|
||||
def _resolve_cleanup_params(override: AdminCleanupRequest) -> dict:
|
||||
"""合并 config.yaml 默认值 + 请求显式覆盖。
|
||||
|
||||
显式传 0 / False 也算覆盖(用 ``is not None`` 判定),方便测「保留 0 天 =
|
||||
全删」这种极端场景。
|
||||
"""
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
raw = getattr(get_app_config(), "ai_writing", None) or {}
|
||||
cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
|
||||
|
||||
return {
|
||||
"retention_days": (
|
||||
override.retention_days if override.retention_days is not None
|
||||
else int(cfg.get("retention_days", 7))
|
||||
),
|
||||
"only_finished": (
|
||||
override.only_finished if override.only_finished is not None
|
||||
else bool(cfg.get("delete_only_finished", False))
|
||||
),
|
||||
"batch_size": (
|
||||
override.batch_size if override.batch_size is not None
|
||||
else int(cfg.get("batch_size", 500))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/admin/cleanup", response_model=AdminCleanupResponse)
|
||||
async def admin_cleanup(
|
||||
body: AdminCleanupRequest,
|
||||
request: Request,
|
||||
) -> AdminCleanupResponse:
|
||||
"""**仅管理员** 手动触发一次会话清理(不等凌晨 3 点定时调度)。
|
||||
|
||||
用途:
|
||||
- 验证 cleanup 配置改动是否生效
|
||||
- 内网紧急清理(磁盘满 / MySQL 行数过多)
|
||||
- 调试:传 ``retention_days: 0`` 一刷拉清空,方便重新跑回归
|
||||
|
||||
所有参数留空 → 沿用 config.yaml 的 ``ai_writing.cleanup`` 段默认。显式
|
||||
传值会覆盖配置但**不修改 config.yaml**(一次性参数,下次定时调度还是用
|
||||
配置文件的值)。
|
||||
"""
|
||||
import time
|
||||
|
||||
from app.gateway.ai_writing_cleanup import run_cleanup_once
|
||||
|
||||
await _require_admin(request)
|
||||
params = _resolve_cleanup_params(body)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = await run_cleanup_once(**params)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing admin cleanup failed: params=%s", params)
|
||||
# 抛 500 是合理的 —— 管理员触发清理失败需要看到失败原因,跟「业务
|
||||
# 接口被失败拖累」是两回事
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"清理执行失败: {type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
logger.info(
|
||||
"ai_writing admin cleanup done: params=%s deleted_db=%d deleted_cp=%d elapsed=%.2fs",
|
||||
params,
|
||||
result["deleted_db"],
|
||||
result["deleted_checkpoints"],
|
||||
elapsed,
|
||||
)
|
||||
return AdminCleanupResponse(
|
||||
deleted_db=result["deleted_db"],
|
||||
deleted_checkpoints=result["deleted_checkpoints"],
|
||||
skipped_active=result.get("skipped_active", 0),
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
params=params,
|
||||
)
|
||||
|
||||
|
||||
# ── 内部辅助 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _stream_graph_segment(
|
||||
@@ -623,6 +864,105 @@ def _event_type(evt: dict) -> str:
|
||||
return t
|
||||
|
||||
|
||||
async def _load_graph_snapshot(session_id: str):
|
||||
"""读取 AI 写作 Graph 在该 session 上的状态快照(失败/无状态返回 None)。
|
||||
|
||||
Graph 的 checkpointer 现在按 config.yaml 持久化到磁盘(默认 SQLite),
|
||||
所以容器重启后只要数据卷还在,仍然能恢复出来。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
|
||||
graph = await get_ai_writing_graph()
|
||||
snapshot = await graph.aget_state(_make_config(session_id))
|
||||
except Exception:
|
||||
logger.exception("ai_writing: aget_state failed during recovery: session=%s", session_id)
|
||||
return None
|
||||
if not snapshot or not getattr(snapshot, "values", None):
|
||||
return None
|
||||
return snapshot
|
||||
|
||||
|
||||
def _extract_interrupt(snapshot) -> dict | None:
|
||||
"""从快照里取出当前挂起的 interrupt 负载(即 await_user 的数据)。"""
|
||||
for task in getattr(snapshot, "tasks", None) or ():
|
||||
for intr in getattr(task, "interrupts", None) or ():
|
||||
value = getattr(intr, "value", None)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
async def _recover_session(session_id: str, request: Request | None = None) -> str:
|
||||
"""内存会话注册表丢失时,尝试从 checkpointer 重建会话。
|
||||
|
||||
返回:
|
||||
- "interrupted" —— 图正停在某个干预点,已重建队列并预置 await_user 事件,可继续 resume
|
||||
- "finished" —— 图已完成,已重建队列并预置 done 事件(+草稿)
|
||||
- "unrecoverable" —— 无可恢复状态(进程/容器重启过,或 session 从未存在)
|
||||
|
||||
幂等:内存里已有队列时直接当作 "interrupted" 返回。整段在 `_recovery_lock`
|
||||
下执行,避免并发的 /stream 与 /resume 各自重建出不同的队列对象。
|
||||
"""
|
||||
async with _recovery_lock:
|
||||
if session_id in _event_queues:
|
||||
return "interrupted"
|
||||
|
||||
snapshot = await _load_graph_snapshot(session_id)
|
||||
if snapshot is None:
|
||||
return "unrecoverable"
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
_event_queues[session_id] = queue
|
||||
_session_interventions.setdefault(session_id, [])
|
||||
_session_model_names.setdefault(session_id, "")
|
||||
_session_article_types.setdefault(session_id, [])
|
||||
|
||||
# 尽量恢复 DB 仓库引用,使后续 resume 仍能持久化草稿/审核结果
|
||||
if request is not None and session_id not in _session_repos:
|
||||
try:
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_current_user
|
||||
|
||||
repo = get_ai_writing_session_store(request)
|
||||
if repo is not None:
|
||||
_session_repos[session_id] = repo
|
||||
_session_user_ids[session_id] = await get_current_user(request)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to restore repo during recovery: session=%s", session_id)
|
||||
|
||||
interrupt_val = _extract_interrupt(snapshot)
|
||||
if interrupt_val is not None:
|
||||
event = ("await_user", {**interrupt_val, "session_id": session_id})
|
||||
await queue.put(event)
|
||||
_session_pending_pause[session_id] = event
|
||||
logger.info("ai_writing: recovered interrupted session from checkpointer: session=%s", session_id)
|
||||
return "interrupted"
|
||||
|
||||
values = snapshot.values or {}
|
||||
if not getattr(snapshot, "next", None) or values.get("status") == "done":
|
||||
draft = values.get("current_draft") or {}
|
||||
if draft.get("full_markdown"):
|
||||
await queue.put(("draft_ready", {
|
||||
"session_id": session_id,
|
||||
"title": draft.get("title", ""),
|
||||
"markdown": draft.get("full_markdown", ""),
|
||||
"revision_count": draft.get("revision_count", 0),
|
||||
}))
|
||||
await queue.put(("done", {"session_id": session_id, "message": "写作已完成"}))
|
||||
await queue.put(None) # 哨兵:让 SSE 生成器读完即关闭
|
||||
logger.info("ai_writing: recovered finished session from checkpointer: session=%s", session_id)
|
||||
return "finished"
|
||||
|
||||
# 既不在干预点、也未结束:上次运行多半被中断在某个节点,无法续推
|
||||
await queue.put(("error", {
|
||||
"session_id": session_id,
|
||||
"message": "写作流程已中断,请重新发起写作。",
|
||||
}))
|
||||
await queue.put(None)
|
||||
logger.warning("ai_writing: session checkpoint stuck mid-flow, not resumable: session=%s", session_id)
|
||||
return "finished"
|
||||
|
||||
|
||||
async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
|
||||
"""从队列消费事件,推送为 SSE 帧。"""
|
||||
# 等待队列创建(最多 5 秒)
|
||||
@@ -631,10 +971,24 @@ async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
yield _sse("error", {"message": "Session not found"})
|
||||
return
|
||||
# 内存里没有这个会话队列。不要直接报「Session not found」——
|
||||
# 先尝试从 checkpointer 恢复(图可能仍停在某个干预点 / 已完成)。
|
||||
recovery = await _recover_session(session_id)
|
||||
if recovery == "unrecoverable":
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
yield _sse("error", {
|
||||
"session_id": session_id,
|
||||
"message": "会话状态已丢失(服务可能已重启),请重新发起写作。",
|
||||
})
|
||||
return
|
||||
# 恢复成功:_event_queues[session_id] 已重建并预置事件,继续走正常消费逻辑。
|
||||
|
||||
queue = _event_queues[session_id]
|
||||
queue = _event_queues.get(session_id)
|
||||
if queue is None:
|
||||
# 极端兜底:恢复后又被并发清理
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
yield _sse("error", {"session_id": session_id, "message": "会话状态已丢失,请重新发起写作。"})
|
||||
return
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
|
||||
# Re-inject the saved await_user event if the queue is empty and a pending pause exists.
|
||||
|
||||
Reference in new issue
Block a user