Merge remote-tracking branch 'origin/master'

This commit is contained in:
吉鑫升 committed 2026-05-20 17:06:06 +08:00
commit 313290b22c
21 files changed
+6764 -2745

No files matched your search

@@ -1,3 +1,15 @@
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
from .graph import (
AI_WRITING_GRAPH,
build_ai_writing_graph,
get_ai_writing_checkpointer,
get_ai_writing_graph,
shutdown_ai_writing_graph,
)
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
__all__ = [
"AI_WRITING_GRAPH",
"build_ai_writing_graph",
"get_ai_writing_graph",
"get_ai_writing_checkpointer",
"shutdown_ai_writing_graph",
]
@@ -1,7 +1,8 @@
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
from __future__ import annotations
from langgraph.checkpoint.memory import MemorySaver
import logging
from langgraph.graph import END, StateGraph
from langgraph.types import interrupt
@@ -12,6 +13,8 @@ from .nodes.writer_outline import writer_outline_node
from .nodes.writer_draft import writer_draft_node
from .nodes.editor import editor_node
logger = logging.getLogger(__name__)
# ── 干预节点 ──────────────────────────────────────────────────────────────────
async def pause_material_confirm(state: AIWritingState) -> dict:
@@ -298,7 +301,16 @@ def after_review_pause(state: AIWritingState) -> str:
# ── 组装 Graph ────────────────────────────────────────────────────────────────
def build_ai_writing_graph():
def build_ai_writing_graph(checkpointer=None):
"""构建并 compile AI 写作 graph。
Args:
checkpointer: 可选的 LangGraph checkpointer。
- ``None`` → 退回 ``MemorySaver`` —— 仅供测试 / 开发使用。
进程或容器重启后会话状态全部丢失,前端拿着旧 session_id 调
``/resume`` 会得到 404。
- 其他 → 传入的 (Async)BaseCheckpointSaver 实例,用于持久化。
"""
g = StateGraph(AIWritingState)
g.add_node("intent_parser", intent_parser_node)
@@ -339,8 +351,188 @@ def build_ai_writing_graph():
g.add_conditional_edges("pause_review", after_review_pause,
{"end": END, "writer_draft": "writer_draft"})
checkpointer = MemorySaver()
if checkpointer is None:
# MemorySaver 仅作为 fallback —— 提示一下,正式部署应通过
# get_ai_writing_graph() 拿到带持久化 checkpointer 的 graph。
from langgraph.checkpoint.memory import MemorySaver
checkpointer = MemorySaver()
return g.compile(checkpointer=checkpointer)
AI_WRITING_GRAPH = build_ai_writing_graph()
# ── 持久化 checkpointer 单例(懒加载)─────────────────────────────────────────
# 模块级 graph 槽位。import 时为 None;首次 ``get_ai_writing_graph()`` 调用时
# 按 config.yaml 的 checkpointer 段懒构建出真正的 graph。
# 测试可通过 ``monkeypatch.setattr(graph_mod, "AI_WRITING_GRAPH", stub)``
# 直接注入桩;本模块所有访问点都走 ``get_ai_writing_graph()`` 这一入口,会优先
# 返回 module attribute 的值,保留测试桩注入路径。
AI_WRITING_GRAPH = None
# 持有 async checkpointer(底层 aiosqlite Connection 或 psycopg pool 的所有者)。
# 进程生命周期内常驻;shutdown 时由 ``shutdown_ai_writing_graph`` 显式关闭。
_async_checkpointer_singleton = None
def _fallback_to_memory_saver(reason: str):
"""退回 InMemorySaver 的统一兜底入口。
把 reason 打到 warning 日志里方便运维定位。InMemorySaver 进程重启会丢
状态,但**当次写作能跑完** —— 我们的首要目标是「写作流程能继续」,
持久化是锦上添花。
"""
from langgraph.checkpoint.memory import InMemorySaver
logger.warning(
"ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver。原因: %s",
reason,
)
return InMemorySaver()
async def _get_async_checkpointer():
"""按 config.yaml 的 checkpointer 段返回一个 async checkpointer 实例。
**绝不抛异常** —— 任何失败一律退回 InMemorySaver。这是为了保证「写作
流程能继续」这个首要目标:持久化坏了用户至少还能写完当次稿,重启会丢
会话但不影响当下创作。
优先级:
1. ``checkpointer:`` 段(``type=sqlite|postgres|memory``)
2. fallback 到 ``InMemorySaver`` 并打 warning
返回的 saver 与主 agent 公用同一 sqlite 文件 —— LangGraph 的 schema
用 ``(thread_id, checkpoint_ns)`` 做主键,AI 写作的 thread_id 是
会话 uuid,跟主 agent 的 LangGraph thread_id 不会撞。
"""
global _async_checkpointer_singleton
if _async_checkpointer_singleton is not None:
return _async_checkpointer_singleton
try:
from deerflow.config.checkpointer_config import get_checkpointer_config
cp_cfg = get_checkpointer_config()
except Exception:
logger.exception("ai_writing: 加载 checkpointer 配置失败")
_async_checkpointer_singleton = _fallback_to_memory_saver("加载 config 抛异常")
return _async_checkpointer_singleton
if cp_cfg is None:
_async_checkpointer_singleton = _fallback_to_memory_saver(
"config.yaml 未配置 checkpointer 段"
)
return _async_checkpointer_singleton
if cp_cfg.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
_async_checkpointer_singleton = InMemorySaver()
logger.info("ai_writing: 配置 type=memory,使用 InMemorySaver")
return _async_checkpointer_singleton
if cp_cfg.type == "sqlite":
try:
import aiosqlite
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.store._sqlite_utils import (
ensure_sqlite_parent_dir,
resolve_sqlite_conn_str,
)
conn_str = resolve_sqlite_conn_str(
cp_cfg.connection_string or "ai_writing_checkpoints.db"
)
ensure_sqlite_parent_dir(conn_str)
# 直接持有 aiosqlite.Connection —— 进程生命周期,shutdown 时显式关。
conn = await aiosqlite.connect(conn_str)
saver = AsyncSqliteSaver(conn)
await saver.setup()
_async_checkpointer_singleton = saver
logger.info("ai_writing: 使用 AsyncSqliteSaver (%s)", conn_str)
return saver
except Exception as exc:
# sqlite 失败原因可能是:依赖未装 / 磁盘满 / 文件权限 / 路径不存在 /
# 文件被外部进程独占锁。任何一个都不能让 AI 写作整个不可用。
logger.exception(
"ai_writing: AsyncSqliteSaver 初始化失败,conn_str=%r",
getattr(cp_cfg, "connection_string", None),
)
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"sqlite 初始化失败: {type(exc).__name__}: {exc}"
)
return _async_checkpointer_singleton
if cp_cfg.type == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from psycopg_pool import AsyncConnectionPool
if not cp_cfg.connection_string:
raise ValueError("checkpointer.connection_string 是 postgres 必填项")
pool = AsyncConnectionPool(cp_cfg.connection_string, open=False)
await pool.open()
saver = AsyncPostgresSaver(pool)
await saver.setup()
_async_checkpointer_singleton = saver
logger.info("ai_writing: 使用 AsyncPostgresSaver")
return saver
except Exception as exc:
logger.exception("ai_writing: AsyncPostgresSaver 初始化失败")
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"postgres 初始化失败: {type(exc).__name__}: {exc}"
)
return _async_checkpointer_singleton
# 未知 type —— config 配错也不能让服务挂
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"未知 checkpointer.type={cp_cfg.type!r}"
)
return _async_checkpointer_singleton
async def get_ai_writing_graph():
"""懒加载并返回 AI 写作 graph 单例。
第一次调用按 config 选定的 checkpointer 构建 graph 并缓存;后续调用直接
返回缓存。测试可在调用前通过 ``monkeypatch.setattr`` 直接替换模块级
``AI_WRITING_GRAPH`` —— 本函数会优先返回已设的值。
"""
global AI_WRITING_GRAPH
if AI_WRITING_GRAPH is not None:
return AI_WRITING_GRAPH
checkpointer = await _get_async_checkpointer()
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
return AI_WRITING_GRAPH
def get_ai_writing_checkpointer():
"""返回当前已实例化的 checkpointer(清理任务用它来删 thread)。
若尚未懒构建过则返回 None。
"""
return _async_checkpointer_singleton
async def shutdown_ai_writing_graph() -> None:
"""关闭 saver 持有的底层连接,并清空 graph 单例。Lifespan shutdown 调用。"""
global AI_WRITING_GRAPH, _async_checkpointer_singleton
saver = _async_checkpointer_singleton
_async_checkpointer_singleton = None
AI_WRITING_GRAPH = None
if saver is None:
return
# aiosqlite saver: saver.conn 是 aiosqlite.Connection
conn = getattr(saver, "conn", None)
if conn is not None and hasattr(conn, "close"):
try:
await conn.close()
except Exception:
logger.warning("ai_writing: 关闭 checkpointer connection 失败", exc_info=True)
# postgres saver: saver.pool 是 AsyncConnectionPool
pool = getattr(saver, "pool", None)
if pool is not None and hasattr(pool, "close"):
try:
await pool.close()
except Exception:
logger.warning("ai_writing: 关闭 checkpointer pool 失败", exc_info=True)
@@ -8,6 +8,7 @@ from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import PortableLongText
class AIWritingSessionRow(Base):
@@ -22,7 +23,8 @@ class AIWritingSessionRow(Base):
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
@@ -160,3 +160,38 @@ class AIWritingSessionRepository:
await session.delete(row)
await session.commit()
return True
async def list_older_than(
self,
cutoff: datetime,
*,
only_finished: bool = False,
limit: int = 500,
) -> list[str]:
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
Args:
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
only_finished: True 时只清 status in (done/error),False 不看状态
limit: 单批最多返回多少条,用于分批处理避免大事务
"""
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
if only_finished:
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [row[0] for row in result.all()]
async def delete_by_ids(self, session_ids: list[str]) -> int:
"""按 id 列表批量删除。返回实际删除的行数。"""
if not session_ids:
return 0
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
async with self._sf() as session:
result = await session.execute(stmt)
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,64 @@
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
The ``transcript`` column stores the full AI-writing conversation timeline as a
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
a transcript with many progress events overflows it and the save endpoint
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
Revision ID: 20260519_02
Revises: 20260519_01
Create Date: 2026-05-19
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260519_02"
down_revision = "20260519_01"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
bind = op.get_bind()
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=LONGTEXT(),
existing_nullable=True,
)
def downgrade() -> None:
bind = op.get_bind()
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
# is a developer-only path.
import sqlalchemy as sa
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=sa.Text(),
existing_nullable=True,
)
@@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.types import TypeDecorator
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
@@ -39,6 +40,25 @@ class PortableJSON(TypeDecorator):
return json.loads(value)
class PortableLongText(TypeDecorator):
"""A large text column.
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
already unbounded.
"""
impl = Text
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == "mysql":
return dialect.type_descriptor(LONGTEXT())
return dialect.type_descriptor(Text())
class BeijingDateTime(TypeDecorator):
"""Store timestamps as Beijing (UTC+8) wall-clock time.