Merge remote-tracking branch 'origin/master'
This commit is contained in:
commit
313290b22c
21 files changed
+6764
-2745
No files matched your search
+14
-2
@@ -1,3 +1,15 @@
|
||||
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
|
||||
from .graph import (
|
||||
AI_WRITING_GRAPH,
|
||||
build_ai_writing_graph,
|
||||
get_ai_writing_checkpointer,
|
||||
get_ai_writing_graph,
|
||||
shutdown_ai_writing_graph,
|
||||
)
|
||||
|
||||
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
|
||||
__all__ = [
|
||||
"AI_WRITING_GRAPH",
|
||||
"build_ai_writing_graph",
|
||||
"get_ai_writing_graph",
|
||||
"get_ai_writing_checkpointer",
|
||||
"shutdown_ai_writing_graph",
|
||||
]
|
||||
+196
-4
@@ -1,7 +1,8 @@
|
||||
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
import logging
|
||||
|
||||
from langgraph.graph import END, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
@@ -12,6 +13,8 @@ from .nodes.writer_outline import writer_outline_node
|
||||
from .nodes.writer_draft import writer_draft_node
|
||||
from .nodes.editor import editor_node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 干预节点 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
@@ -298,7 +301,16 @@ def after_review_pause(state: AIWritingState) -> str:
|
||||
|
||||
# ── 组装 Graph ────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_ai_writing_graph():
|
||||
def build_ai_writing_graph(checkpointer=None):
|
||||
"""构建并 compile AI 写作 graph。
|
||||
|
||||
Args:
|
||||
checkpointer: 可选的 LangGraph checkpointer。
|
||||
- ``None`` → 退回 ``MemorySaver`` —— 仅供测试 / 开发使用。
|
||||
进程或容器重启后会话状态全部丢失,前端拿着旧 session_id 调
|
||||
``/resume`` 会得到 404。
|
||||
- 其他 → 传入的 (Async)BaseCheckpointSaver 实例,用于持久化。
|
||||
"""
|
||||
g = StateGraph(AIWritingState)
|
||||
|
||||
g.add_node("intent_parser", intent_parser_node)
|
||||
@@ -339,8 +351,188 @@ def build_ai_writing_graph():
|
||||
g.add_conditional_edges("pause_review", after_review_pause,
|
||||
{"end": END, "writer_draft": "writer_draft"})
|
||||
|
||||
checkpointer = MemorySaver()
|
||||
if checkpointer is None:
|
||||
# MemorySaver 仅作为 fallback —— 提示一下,正式部署应通过
|
||||
# get_ai_writing_graph() 拿到带持久化 checkpointer 的 graph。
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
|
||||
checkpointer = MemorySaver()
|
||||
return g.compile(checkpointer=checkpointer)
|
||||
|
||||
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph()
|
||||
# ── 持久化 checkpointer 单例(懒加载)─────────────────────────────────────────
|
||||
|
||||
# 模块级 graph 槽位。import 时为 None;首次 ``get_ai_writing_graph()`` 调用时
|
||||
# 按 config.yaml 的 checkpointer 段懒构建出真正的 graph。
|
||||
# 测试可通过 ``monkeypatch.setattr(graph_mod, "AI_WRITING_GRAPH", stub)``
|
||||
# 直接注入桩;本模块所有访问点都走 ``get_ai_writing_graph()`` 这一入口,会优先
|
||||
# 返回 module attribute 的值,保留测试桩注入路径。
|
||||
AI_WRITING_GRAPH = None
|
||||
|
||||
# 持有 async checkpointer(底层 aiosqlite Connection 或 psycopg pool 的所有者)。
|
||||
# 进程生命周期内常驻;shutdown 时由 ``shutdown_ai_writing_graph`` 显式关闭。
|
||||
_async_checkpointer_singleton = None
|
||||
|
||||
|
||||
def _fallback_to_memory_saver(reason: str):
|
||||
"""退回 InMemorySaver 的统一兜底入口。
|
||||
|
||||
把 reason 打到 warning 日志里方便运维定位。InMemorySaver 进程重启会丢
|
||||
状态,但**当次写作能跑完** —— 我们的首要目标是「写作流程能继续」,
|
||||
持久化是锦上添花。
|
||||
"""
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
logger.warning(
|
||||
"ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver。原因: %s",
|
||||
reason,
|
||||
)
|
||||
return InMemorySaver()
|
||||
|
||||
|
||||
async def _get_async_checkpointer():
|
||||
"""按 config.yaml 的 checkpointer 段返回一个 async checkpointer 实例。
|
||||
|
||||
**绝不抛异常** —— 任何失败一律退回 InMemorySaver。这是为了保证「写作
|
||||
流程能继续」这个首要目标:持久化坏了用户至少还能写完当次稿,重启会丢
|
||||
会话但不影响当下创作。
|
||||
|
||||
优先级:
|
||||
1. ``checkpointer:`` 段(``type=sqlite|postgres|memory``)
|
||||
2. fallback 到 ``InMemorySaver`` 并打 warning
|
||||
|
||||
返回的 saver 与主 agent 公用同一 sqlite 文件 —— LangGraph 的 schema
|
||||
用 ``(thread_id, checkpoint_ns)`` 做主键,AI 写作的 thread_id 是
|
||||
会话 uuid,跟主 agent 的 LangGraph thread_id 不会撞。
|
||||
"""
|
||||
global _async_checkpointer_singleton
|
||||
if _async_checkpointer_singleton is not None:
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
try:
|
||||
from deerflow.config.checkpointer_config import get_checkpointer_config
|
||||
|
||||
cp_cfg = get_checkpointer_config()
|
||||
except Exception:
|
||||
logger.exception("ai_writing: 加载 checkpointer 配置失败")
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver("加载 config 抛异常")
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg is None:
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
"config.yaml 未配置 checkpointer 段"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
_async_checkpointer_singleton = InMemorySaver()
|
||||
logger.info("ai_writing: 配置 type=memory,使用 InMemorySaver")
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "sqlite":
|
||||
try:
|
||||
import aiosqlite
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from deerflow.runtime.store._sqlite_utils import (
|
||||
ensure_sqlite_parent_dir,
|
||||
resolve_sqlite_conn_str,
|
||||
)
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(
|
||||
cp_cfg.connection_string or "ai_writing_checkpoints.db"
|
||||
)
|
||||
ensure_sqlite_parent_dir(conn_str)
|
||||
# 直接持有 aiosqlite.Connection —— 进程生命周期,shutdown 时显式关。
|
||||
conn = await aiosqlite.connect(conn_str)
|
||||
saver = AsyncSqliteSaver(conn)
|
||||
await saver.setup()
|
||||
_async_checkpointer_singleton = saver
|
||||
logger.info("ai_writing: 使用 AsyncSqliteSaver (%s)", conn_str)
|
||||
return saver
|
||||
except Exception as exc:
|
||||
# sqlite 失败原因可能是:依赖未装 / 磁盘满 / 文件权限 / 路径不存在 /
|
||||
# 文件被外部进程独占锁。任何一个都不能让 AI 写作整个不可用。
|
||||
logger.exception(
|
||||
"ai_writing: AsyncSqliteSaver 初始化失败,conn_str=%r",
|
||||
getattr(cp_cfg, "connection_string", None),
|
||||
)
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"sqlite 初始化失败: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
if not cp_cfg.connection_string:
|
||||
raise ValueError("checkpointer.connection_string 是 postgres 必填项")
|
||||
pool = AsyncConnectionPool(cp_cfg.connection_string, open=False)
|
||||
await pool.open()
|
||||
saver = AsyncPostgresSaver(pool)
|
||||
await saver.setup()
|
||||
_async_checkpointer_singleton = saver
|
||||
logger.info("ai_writing: 使用 AsyncPostgresSaver")
|
||||
return saver
|
||||
except Exception as exc:
|
||||
logger.exception("ai_writing: AsyncPostgresSaver 初始化失败")
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"postgres 初始化失败: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
# 未知 type —— config 配错也不能让服务挂
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"未知 checkpointer.type={cp_cfg.type!r}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
|
||||
async def get_ai_writing_graph():
|
||||
"""懒加载并返回 AI 写作 graph 单例。
|
||||
|
||||
第一次调用按 config 选定的 checkpointer 构建 graph 并缓存;后续调用直接
|
||||
返回缓存。测试可在调用前通过 ``monkeypatch.setattr`` 直接替换模块级
|
||||
``AI_WRITING_GRAPH`` —— 本函数会优先返回已设的值。
|
||||
"""
|
||||
global AI_WRITING_GRAPH
|
||||
if AI_WRITING_GRAPH is not None:
|
||||
return AI_WRITING_GRAPH
|
||||
checkpointer = await _get_async_checkpointer()
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
|
||||
return AI_WRITING_GRAPH
|
||||
|
||||
|
||||
def get_ai_writing_checkpointer():
|
||||
"""返回当前已实例化的 checkpointer(清理任务用它来删 thread)。
|
||||
|
||||
若尚未懒构建过则返回 None。
|
||||
"""
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
|
||||
async def shutdown_ai_writing_graph() -> None:
|
||||
"""关闭 saver 持有的底层连接,并清空 graph 单例。Lifespan shutdown 调用。"""
|
||||
global AI_WRITING_GRAPH, _async_checkpointer_singleton
|
||||
saver = _async_checkpointer_singleton
|
||||
_async_checkpointer_singleton = None
|
||||
AI_WRITING_GRAPH = None
|
||||
if saver is None:
|
||||
return
|
||||
# aiosqlite saver: saver.conn 是 aiosqlite.Connection
|
||||
conn = getattr(saver, "conn", None)
|
||||
if conn is not None and hasattr(conn, "close"):
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
logger.warning("ai_writing: 关闭 checkpointer connection 失败", exc_info=True)
|
||||
# postgres saver: saver.pool 是 AsyncConnectionPool
|
||||
pool = getattr(saver, "pool", None)
|
||||
if pool is not None and hasattr(pool, "close"):
|
||||
try:
|
||||
await pool.close()
|
||||
except Exception:
|
||||
logger.warning("ai_writing: 关闭 checkpointer pool 失败", exc_info=True)
|
||||
+4
-2
@@ -8,6 +8,7 @@ from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import PortableLongText
|
||||
|
||||
|
||||
class AIWritingSessionRow(Base):
|
||||
@@ -22,7 +23,8 @@ class AIWritingSessionRow(Base):
|
||||
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
|
||||
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
|
||||
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
|
||||
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
|
||||
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+36
-1
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
@@ -160,3 +160,38 @@ class AIWritingSessionRepository:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_older_than(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
only_finished: bool = False,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
|
||||
|
||||
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
|
||||
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
|
||||
|
||||
Args:
|
||||
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
|
||||
only_finished: True 时只清 status in (done/error),False 不看状态
|
||||
limit: 单批最多返回多少条,用于分批处理避免大事务
|
||||
"""
|
||||
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
|
||||
if only_finished:
|
||||
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
async def delete_by_ids(self, session_ids: list[str]) -> int:
|
||||
"""按 id 列表批量删除。返回实际删除的行数。"""
|
||||
if not session_ids:
|
||||
return 0
|
||||
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
|
||||
|
||||
The ``transcript`` column stores the full AI-writing conversation timeline as a
|
||||
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
|
||||
a transcript with many progress events overflows it and the save endpoint
|
||||
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
|
||||
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
|
||||
|
||||
Revision ID: 20260519_02
|
||||
Revises: 20260519_01
|
||||
Create Date: 2026-05-19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
|
||||
|
||||
revision = "20260519_02"
|
||||
down_revision = "20260519_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=LONGTEXT(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
|
||||
# is a developer-only path.
|
||||
import sqlalchemy as sa
|
||||
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
@@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
|
||||
@@ -39,6 +40,25 @@ class PortableJSON(TypeDecorator):
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class PortableLongText(TypeDecorator):
|
||||
"""A large text column.
|
||||
|
||||
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
|
||||
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
|
||||
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
|
||||
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
|
||||
already unbounded.
|
||||
"""
|
||||
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(LONGTEXT())
|
||||
return dialect.type_descriptor(Text())
|
||||
|
||||
|
||||
class BeijingDateTime(TypeDecorator):
|
||||
"""Store timestamps as Beijing (UTC+8) wall-clock time.
|
||||
|
||||
|
||||
Reference in new issue
Block a user