This commit is contained in:
liuq_1233 committed 2026-05-24 10:22:01 +08:00
commit 9ffe28f049
223 files changed
+28152 -13879

No files matched your search

+1 -1
View File
@@ -58,4 +58,4 @@ ELEMENT_STATE_FILE=/app/backend/.deer-flow/element-bot-token
# PYTHON_PACKAGE_INDEX_URL=http://your-python-mirror/simple
# PYTHON_EXTRA_INDEX_URL=
# PYTHON_TRUSTED_HOST=your-python-mirror
MYSQL_DATABASE_URL=mysql+asyncmy://kland:kland%4088124932@192.168.0.123:3306/deerflow?charset=utf8mb4
MYSQL_DATABASE_URL=mysql+asyncmy://deerflow:deerflow123@192.168.0.123:3306/deerflow?charset=utf8mb4
+5 -2
View File
@@ -187,7 +187,7 @@ Lead-agent middlewares are assembled in strict append order across `packages/har
16. **SubagentLimitMiddleware** - Truncates excess `task` tool calls from model response to enforce `MAX_CONCURRENT_SUBAGENTS` limit (optional, if `subagent_enabled`)
17. **LoopDetectionMiddleware** - Detects repeated tool-call loops; hard-stop responses clear both structured `tool_calls` and raw provider tool-call metadata before forcing a final text answer
18. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last)
8a. **ToolMetricsMiddleware** - Appended immediately after `ToolErrorHandlingMiddleware` in `_build_runtime_middlewares`, so it sits _deepest_ in the `wrap_tool_call` chain. Records one `tool_call_metrics` row per tool/skill/MCP invocation — timing duration and bucketing failures into a coarse `error_category` (rate_limited / ip_blocked / auth_error / timeout / network_error / not_found / empty_result / tool_error). It catches both raised exceptions (before #8 converts them) and _self-handled_ errors returned as error `ToolMessage`s (search tools that swallow HTTP 429/403). It also captures the targeted Agent Skill into `tool_call_metrics.skill_name`: the lead agent invokes a skill by `read_file`-ing its `/mnt/skills/<category>/<name>/SKILL.md` path (per the `<skill_system>` prompt block — there is no dedicated "use skill" tool), so `skill_name` is recovered by scanning every tool call's string arguments for the `skills/{public,custom}/{name}/` path shape (`_SKILL_PATH_RE`), plus the explicit `name` argument of `skill_view`/`skill_manage`/`skill_archive`. `skill_view`-specific success detection handles its swallowed failures (`Skill 'x' not found.` returned as a plain string rather than raised). Regression coverage: `tests/test_tool_metrics_middleware.py`. Writes are best-effort fire-and-forget. **Per-run skill merging**: `SqlToolMetricsStore.record` upserts skill rows by `(run_id, skill_name)` instead of inserting one row per call — a skill touched by several tool calls / retries inside one run collapses to a single row. Any success in the run wins (a skill that failed then finally succeeded is recorded as one `success`); a skill that only ever failed yields one `error` row whose `error_message` accumulates every attempt's message; durations are summed. Non-skill tool calls, and skill calls with no `run_id`, keep one row each. The read→merge→write is guarded by a per-event-loop `asyncio.Lock`. Surfaced by the admin `GET /api/admin/tool-metrics` router (which has a `kind=skill` filter and a per-skill `/summary` breakdown).
8a. **ToolMetricsMiddleware** - Appended immediately after `ToolErrorHandlingMiddleware` in `_build_runtime_middlewares`, so it sits _deepest_ in the `wrap_tool_call` chain. Records one `tool_call_metrics` row per tool/skill/MCP invocation — timing duration and bucketing failures into a coarse `error_category` (rate*limited / ip_blocked / auth_error / timeout / network_error / not_found / empty_result / tool_error). It catches both raised exceptions (before #8 converts them) and \_self-handled* errors returned as error `ToolMessage`s (search tools that swallow HTTP 429/403). It also captures the targeted Agent Skill into `tool_call_metrics.skill_name`: the lead agent invokes a skill by `read_file`-ing its `/mnt/skills/<category>/<name>/SKILL.md` path (per the `<skill_system>` prompt block — there is no dedicated "use skill" tool), so `skill_name` is recovered by scanning every tool call's string arguments for the `skills/{public,custom}/{name}/` path shape (`_SKILL_PATH_RE`), plus the explicit `name` argument of `skill_view`/`skill_manage`/`skill_archive`. `skill_view`-specific success detection handles its swallowed failures (`Skill 'x' not found.` returned as a plain string rather than raised). Regression coverage: `tests/test_tool_metrics_middleware.py`. Writes are best-effort fire-and-forget. **Per-run skill merging**: `SqlToolMetricsStore.record` upserts skill rows by `(run_id, skill_name)` instead of inserting one row per call — a skill touched by several tool calls / retries inside one run collapses to a single row. Any success in the run wins (a skill that failed then finally succeeded is recorded as one `success`); a skill that only ever failed yields one `error` row whose `error_message` accumulates every attempt's message; durations are summed. Non-skill tool calls, and skill calls with no `run_id`, keep one row each. The read→merge→write is guarded by a per-event-loop `asyncio.Lock`. Surfaced by the admin `GET /api/admin/tool-metrics` router (which has a `kind=skill` filter and a per-skill `/summary` breakdown).
19. **PromptPrefixMiddleware** - Injects the admin-configured prompt prefix into the model request only. The prefix is tagged onto the first human message's `additional_kwargs['prompt_prefix']` by the Gateway (`app.gateway.services`) and prepended to content here at call time, so the persisted/displayed message stays prefix-free
20. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
21. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
@@ -249,7 +249,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
| **Skills** (`/api/skills`) | `GET /` - list skills (optional `search` fuzzy-name + `tag_id` filters; each item carries `tags` + owner-authored `detail`); `GET /{name}` - details; `PUT /{name}` - update enabled; `PUT /{name}/detail` - set the long-form `detail` text (custom skill: owner or admin; built-in/public: admin only — materializes a `skills` row on demand); `GET /{name}/content` - raw SKILL.md source, restricted to an admin or the publisher (custom-skill owner); other users get 403 and see only description + detail; `POST /install` - install from .skill archive (accepts standard optional frontmatter like `version`, `author`, `compatibility`) |
| **Skills** (`/api/skills`) | `GET /` - list skills (optional `search` fuzzy-name + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags` + owner-authored `detail`); `GET /{name}` - details; `PUT /{name}` - update enabled; `PUT /{name}/detail` - set the long-form `detail` text (custom skill: owner or admin; built-in/public: admin only — materializes a `skills` row on demand); `GET /{name}/content` - raw SKILL.md source, restricted to an admin or the publisher (custom-skill owner); other users get 403 and see only description + detail; `POST /install` - install from .skill archive (accepts standard optional frontmatter like `version`, `author`, `compatibility`) |
| **Memory** (`/api/memory`) | `GET /` - memory data; `POST /reload` - force reload; `GET /config` - config; `GET /status` - config + data |
| **Agents** (`/api/agents`) | `GET /` - list built-in, owned, and published agents (optional `search` + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags`); `POST /` - create user-owned agent; `GET/PUT/DELETE /{id}` - read/update/delete by stable agent id; `GET /check` - validate non-empty display name (duplicates allowed) |
| **Agents** (`/api/agents`) | `GET /` - list built-in, owned, and published agents (optional `search` + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags` and `featured_order`); `POST /` - create user-owned agent; `GET/PUT/DELETE /{id}` - read/update/delete by stable agent id; `GET /check` - validate non-empty display name (duplicates allowed); `GET /selector` - homepage agent picker (single admin-curated list, visibility-filtered, sorted by `featured_order` asc); `PUT /{id}/featured` - **admin** sets/clears the homepage `featured_order` (must be `published=true` to feature; passing `order: null` clears unconditionally); `PUT /featured/order` - **admin** bulk-reorders the homepage list (body `{agent_ids: [...]}` → indices become 0..N-1) |
| **Tags** (`/api/tags`) | `GET /` - list tags (optional `tag_type` filter; open to all); `POST /` - create tag (admin); `PUT /{id}` / `DELETE /{id}` - update/delete tag (admin); `GET /assignments/{target_type}/{target_id}` - tags on a resource; `PUT /assignments/{target_type}/{target_id}` - replace a resource's tags (admin). Tags are typed `agent`/`skill`/`scheduled_task` and may only tag resources of their own type. |
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
@@ -276,11 +276,14 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` →
Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.agents`). `agents.id` is the stable runtime id and the filesystem directory name under `.deer-flow/agents/{id}/`; `agents.name` is display-only, can contain Chinese characters, and is not unique. `user_id IS NULL` marks built-in agents, user-owned agents are visible only to their owner unless `published = true`, and update/delete operations require ownership. The runtime accepts `agent_id` in context/configurable, and non-default `assistant_id` values are treated as agent ids after visibility checks.
**Homepage selector (agent picker on the new-chat page)** — single admin-curated list backed by `agents.featured_order` (nullable BIGINT). NULL = not featured. Only `published = true` agents can be featured (router enforces on writes); any published agent is eligible regardless of ownership. Cleared with `order: null`. The list rendered to end users is `WHERE featured_order IS NOT NULL` + visibility-filter, sorted by `featured_order ASC`. Admins reorder by sending the desired ids to `PUT /api/agents/featured/order` (assigns indices 0..N-1). BIGINT is used rather than INT so the management UI can seed initial values from `Date.now()` (overflows INT on MySQL).
### Portable Column Types (`deerflow/persistence/types.py`)
Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` / `mysql` backends. **All ORM timestamp/JSON columns must use these, not the raw SQLAlchemy types.**
- **`PortableJSON`** — native `JSON` on SQLite/Postgres; `TEXT` (with manual `json.dumps`/`loads`) on MySQL, where older servers lack JSON support.
- **`PortableLongText`** — a large text column: maps to MySQL `LONGTEXT` (4 GB) and plain `TEXT` on SQLite/Postgres (already unbounded). Use for big JSON/markdown blobs that would overflow MySQL's 64 KB `TEXT` cap — e.g. `ai_writing_sessions.transcript`.
- **`BeijingDateTime`** — stores timestamps as Beijing (UTC+8) wall-clock time. The app records time with `datetime.now(UTC)`, but MySQL `DATETIME` / SQLite have no timezone and persist whatever wall clock the driver serializes — so a raw `DateTime` column would store UTC and read 8 hours behind Beijing. `BeijingDateTime` converts UTC→Beijing on bind and re-attaches `+08:00` on result, so the database holds Beijing time while Python still gets timezone-aware datetimes (instant comparisons stay correct, including in WHERE clauses). PostgreSQL `timestamptz` stores the instant unambiguously, so values pass through unchanged there. Uses a fixed UTC+8 offset (China has no DST), so no `tzdata` dependency.
### Sandbox System (`packages/harness/deerflow/sandbox/`)
@@ -0,0 +1,326 @@
"""AI 写作会话定期清理。
按 ``config.yaml`` 的 ``ai_writing.cleanup`` 段做联动清理:
1. 找出 MySQL ``ai_writing_sessions`` 表里 ``updated_at < now - retention_days``
的会话 id 列表(分批,避免大事务)
2. 对每个 id 调用 LangGraph checkpointer 的 ``adelete_thread()``,删掉
SQLite checkpointer 里对应的 thread 状态
3. 再把 MySQL 行批量删掉
整个流程「先删 checkpointer 再删 DB」是有意为之 —— 即便清理被中途中断,
也只会出现「checkpointer 删了但 DB 没删」的孤儿,下一轮清理会发现这些
DB 行仍然过期,重新尝试清理(checkpointer 删一个不存在的 thread 是 idempotent)。
反过来如果先删 DB,万一 checkpointer 删失败,文件里就永久留着不可达的状态。
多 worker 下用文件锁做 leader election,只有抢到锁的 worker 才会真正跑清理。
"""
from __future__ import annotations
import asyncio
import logging
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
from croniter import croniter
logger = logging.getLogger(__name__)
# ── 文件锁 ────────────────────────────────────────────────────────────────────
def _acquire_lock(lock_path: Path):
"""跨平台非阻塞独占文件锁,多 worker 下做 leader election。
成功 → 返回持有锁的文件句柄(调用方需保持其存活直到主动释放);
抢不到 / 打开失败 → 返回 None。进程退出时 OS 自动释放。
"""
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = open(lock_path, "a+")
except OSError:
logger.exception("ai_writing_cleanup: 打开锁文件失败 path=%s", lock_path)
return None
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
return None
return handle
def _release_lock(handle) -> None:
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError:
pass
finally:
try:
handle.close()
except OSError:
pass
# ── 清理执行 ──────────────────────────────────────────────────────────────────
async def _delete_checkpoint_thread(checkpointer, thread_id: str) -> None:
"""删 checkpointer 里 thread_id 对应的全部状态。失败只记日志,不抛。"""
if checkpointer is None:
return
# LangGraph 异步删法首选 adelete_thread;老版本只有同步 delete_thread。
delete_async = getattr(checkpointer, "adelete_thread", None)
if delete_async is not None:
try:
await delete_async(thread_id)
return
except Exception:
logger.warning(
"ai_writing_cleanup: adelete_thread failed thread_id=%s", thread_id, exc_info=True
)
return
delete_sync = getattr(checkpointer, "delete_thread", None)
if delete_sync is not None:
try:
# 同步方法跑在线程池里,避免堵 event loop
await asyncio.to_thread(delete_sync, thread_id)
except Exception:
logger.warning(
"ai_writing_cleanup: delete_thread failed thread_id=%s", thread_id, exc_info=True
)
async def _get_active_session_ids_langgraph(thread_store) -> set[str]:
"""新路径下「正在跑 / 等用户干预」的 thread_id 集合。
``threads_meta.status`` 由 ``start_run`` / ``run_agent`` 维护:
- ``running``:graph 正在跑(节点中)
- ``interrupted``:graph 暂停在 interrupt 点等用户回头填卡
这两种都视作活跃,**绝不能删**。``idle / success / error`` 才是真正终止状态。
LangGraph Server 路径 + 老路径的 lead_agent 都用同一份 ``thread_store``,所以
这里拿的 active 集合是全部 graph 的;筛 ai_writing 是按 assistant_id 过滤。
"""
if thread_store is None:
return set()
try:
running = await thread_store.search(status="running", user_id=None, limit=1000)
interrupted = await thread_store.search(status="interrupted", user_id=None, limit=1000)
except Exception:
logger.warning("ai_writing_cleanup: 读取 LangGraph 活动 thread 列表失败", exc_info=True)
return set()
# 仅关心 assistant_id='ai_writing' 的 thread —— 别的 graph 不归这套清理管
active: set[str] = set()
for row in running + interrupted:
if row.get("assistant_id") == "ai_writing":
active.add(row["thread_id"])
return active
async def run_cleanup_once(
*,
retention_days: int,
only_finished: bool = False,
batch_size: int = 500,
checkpointer=None,
thread_store=None,
) -> dict:
"""执行一轮清理。返回统计信息。
Args:
checkpointer: 主 LangGraph checkpointer(``app.state.checkpointer``)。
AI 写作的 thread 状态由 LangGraph Server 注入,存在这里 —— admin 路由
+ scheduler loop 都会从 ``app.state`` 取到引用传进来;测试场景下传 None
会让 checkpoint 删除变成 no-op,但 DB 行仍会被清。
thread_store: ``app.state.thread_store``。用来按 ``status in (running,
interrupted)`` 查活跃 thread 列表做活跃保护。
**安全保证**:清理永远跳过仍在跑 / 等用户干预的会话(``threads_meta.status
in (running, interrupted)`` + ``assistant_id='ai_writing'``)。即使 admin
手动触发 ``retention_days=0`` 也强制生效。
幂等 + 可重入 —— 多次调用对结果只会越扫越干净。即便清理途中被中断,下次
再跑会重新把 DB 里仍然过期的行扫到(checkpointer 删一个已不存在的 thread
是 no-op)。
"""
from deerflow.persistence.ai_writing_sessions import AIWritingSessionRepository
from deerflow.persistence.engine import get_session_factory
sf = get_session_factory()
if sf is None:
logger.info("ai_writing_cleanup: persistence backend=memory,跳过清理")
return {"deleted_db": 0, "deleted_checkpoints": 0, "skipped_active": 0}
repo = AIWritingSessionRepository(sf)
active_ids = await _get_active_session_ids_langgraph(thread_store)
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
logger.info(
"ai_writing_cleanup: 开始清理 cutoff=%s only_finished=%s batch_size=%d active=%d",
cutoff.isoformat(),
only_finished,
batch_size,
len(active_ids),
)
total_db = 0
total_cp = 0
total_skipped = 0
while True:
ids = await repo.list_older_than(cutoff, only_finished=only_finished, limit=batch_size)
if not ids:
break
# 关键保护:跳过任何仍在内存里跑的会话
ids_to_delete = [sid for sid in ids if sid not in active_ids]
skipped = len(ids) - len(ids_to_delete)
if skipped:
logger.warning(
"ai_writing_cleanup: 跳过 %d 个仍在跑的会话: %s",
skipped,
[sid for sid in ids if sid in active_ids],
)
total_skipped += skipped
if not ids_to_delete:
# 本批全是活动会话 —— 但 list_older_than 是按 updated_at 升序的,
# 下一批拿出来的更旧,所以这种情况只可能整个 cutoff 区间都被活动
# 会话占满,不用继续往下扫,break
break
# 先删 checkpointer,再删 DB —— 保证「DB 删了 → checkpointer 一定也清干净」
for sid in ids_to_delete:
await _delete_checkpoint_thread(checkpointer, sid)
total_cp += 1
deleted = await repo.delete_by_ids(ids_to_delete)
total_db += deleted
logger.info(
"ai_writing_cleanup: 本批删 %d 行(DB),%d 个 thread(checkpointer),跳过 %d",
deleted,
len(ids_to_delete),
skipped,
)
# 整批比 batch_size 小 → 没有更多了
if len(ids) < batch_size:
break
logger.info(
"ai_writing_cleanup: 清理完成 total_db=%d total_checkpoints=%d skipped_active=%d",
total_db,
total_cp,
total_skipped,
)
return {
"deleted_db": total_db,
"deleted_checkpoints": total_cp,
"skipped_active": total_skipped,
}
# ── 调度循环 ──────────────────────────────────────────────────────────────────
def _next_cron_delay_seconds(cron_expr: str, base: datetime | None = None) -> float:
"""根据 cron 表达式算到下一次触发的秒数。
用 ``datetime.now(UTC)`` 做基准。croniter 接受 naive 时间,这里转成 naive
UTC 喂进去;当然如果用户希望按本地时区跑(凌晨 3 点),需要在外层调整 ——
我们用本地时区基准,等价于 cron 在本地时区跑。
"""
base = base or datetime.now()
nxt = croniter(cron_expr, base).get_next(datetime)
return max(1.0, (nxt - base).total_seconds())
async def cleanup_scheduler_loop(app=None) -> None:
"""常驻后台任务:按 cron 调度跑清理。
多 worker 部署下文件锁做 leader election,没拿到锁的 worker 直接退出。
被 cancel 时 CancelledError 不会被 except Exception 吞掉,能干净退出。
Args:
app: FastAPI 实例,用来从 ``app.state`` 拿主 checkpointer / thread_store
引用,喂给 ``run_cleanup_once`` —— 新路径下 ai_writing 的 thread 状态
在主 checkpointer 里。``app=None`` 时回退到老 checkpointer(仅老路径)。
"""
from deerflow.config.app_config import get_app_config
from deerflow.config.runtime_paths import runtime_home
try:
cfg = get_app_config()
except Exception:
logger.exception("ai_writing_cleanup: 加载 app config 失败,调度器退出")
return
raw = getattr(cfg, "ai_writing", None) or {}
cleanup_cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
if not cleanup_cfg.get("enabled", False):
logger.info("ai_writing_cleanup: cleanup.enabled=false,调度器不启动")
return
cron_expr = str(cleanup_cfg.get("cron", "0 3 * * *"))
retention_days = int(cleanup_cfg.get("retention_days", 7))
only_finished = bool(cleanup_cfg.get("delete_only_finished", False))
batch_size = int(cleanup_cfg.get("batch_size", 500))
# 提前校验 cron 表达式,配错就早 fail 早暴露
try:
croniter(cron_expr, datetime.now())
except Exception:
logger.exception("ai_writing_cleanup: 非法 cron 表达式 %r,调度器退出", cron_expr)
return
lock_handle = _acquire_lock(runtime_home() / ".ai_writing_cleanup.lock")
if lock_handle is None:
logger.info("ai_writing_cleanup: 另一个 worker 已持有调度锁,本 worker 跳过")
return
logger.info(
"ai_writing_cleanup: 已启动 cron=%r retention_days=%d only_finished=%s batch_size=%d",
cron_expr,
retention_days,
only_finished,
batch_size,
)
try:
while True:
delay = _next_cron_delay_seconds(cron_expr)
logger.info("ai_writing_cleanup: 下一次清理 %.0f 秒后", delay)
await asyncio.sleep(delay)
try:
# 每次循环重新读 app.state —— 不缓存,避免 lifespan 中重启 / 重置
# checkpointer 时拿到死引用。app=None 时 main_checkpointer / store
# 也为 None,run_cleanup_once 内部走老 fallback。
main_checkpointer = getattr(getattr(app, "state", None), "checkpointer", None)
main_thread_store = getattr(getattr(app, "state", None), "thread_store", None)
await run_cleanup_once(
retention_days=retention_days,
only_finished=only_finished,
batch_size=batch_size,
checkpointer=main_checkpointer,
thread_store=main_thread_store,
)
except Exception:
logger.exception("ai_writing_cleanup: 本轮清理失败(不影响下一轮)")
finally:
_release_lock(lock_handle)
@@ -1,7 +1,9 @@
import asyncio
import logging
import os
import queue
import sys
import threading
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
@@ -34,7 +36,9 @@ from app.gateway.routers import (
mcp,
memory,
models,
multi_agent,
notifications,
public_scheduled_tasks,
recommended_questions,
runs,
scheduled_tasks,
@@ -244,7 +248,6 @@ async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
async def _run_db_migrations() -> None:
"""Run Alembic migrations at startup so schema changes are applied automatically."""
import asyncio
from pathlib import Path
from alembic import command
@@ -258,17 +261,50 @@ async def _run_db_migrations() -> None:
logger.info("Persistence backend=memory — skipping Alembic migrations")
return
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations")
return
db_url = engine.url.render_as_string(hide_password=False).replace("%", "%%")
migrations_dir = str(Path(_persistence_pkg.__file__).parent / "migrations")
result_queue: queue.Queue[BaseException | None] = queue.Queue(maxsize=1)
def _run_sync() -> None:
cfg = Config()
cfg.set_main_option("script_location", migrations_dir)
cfg.set_main_option("sqlalchemy.url", db_url)
command.upgrade(cfg, "head")
try:
cfg = Config()
cfg.set_main_option("script_location", migrations_dir)
cfg.set_main_option("sqlalchemy.url", db_url)
command.upgrade(cfg, "head")
except BaseException as exc:
result_queue.put(exc)
else:
result_queue.put(None)
try:
await asyncio.get_running_loop().run_in_executor(None, _run_sync)
try:
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
except ValueError:
timeout_s = 60.0
worker = threading.Thread(target=_run_sync, name="deerflow-alembic-upgrade", daemon=True)
worker.start()
try:
deadline = asyncio.get_running_loop().time() + timeout_s
while result_queue.empty():
if asyncio.get_running_loop().time() >= deadline:
raise asyncio.TimeoutError
await asyncio.sleep(0.2)
result = result_queue.get_nowait()
except asyncio.TimeoutError:
logger.error(
"Database migration did not finish within %.0fs; continuing startup on create_all schema. "
"Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
"to skip startup migrations, or inspect MySQL with SHOW FULL PROCESSLIST.",
timeout_s,
)
return
if result is not None:
raise result
logger.info("Database migrations applied (head)")
except Exception:
logger.exception("Database migration failed — schema may be out of date")
@@ -361,10 +397,67 @@ async def _curator_scheduler_loop() -> None:
_release_singleton_lock(lock_handle)
def _warn_if_multi_worker() -> None:
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
维护 run_id → SSE 订阅者的内存 map。多 worker 下,用户在 worker A 发起
``/runs/stream``、稍后在 worker B 调 join/resume,会因 in-process map 互相
看不见而 404 / 卡死。
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
"""
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
# / uvicorn 都认)。
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
logger.error(
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
web_concurrency,
)
return
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
if not sys.platform.startswith("linux"):
return
try:
from pathlib import Path
peers = 0
for entry in Path("/proc").iterdir():
if not entry.name.isdigit():
continue
try:
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
except OSError:
continue
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
peers += 1
if peers > 1:
logger.error(
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
"启用 sticky session。",
peers,
)
except Exception:
# 检测本身失败不算事 —— 这只是个提醒
pass
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
_warn_if_multi_worker()
# Load config and check necessary environment variables at startup
try:
app.state.config = get_app_config()
@@ -384,6 +477,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# Run Alembic migrations before anything else touches the schema.
await _run_db_migrations()
# 预热 recall_budget cache(从 DB 批量加载,使 get_effective_memory_config 同步可读)
try:
from deerflow.config.memory_config import load_recall_budget_cache
_mc_store = getattr(app.state, "memory_config_store", None)
if _mc_store is not None:
_recall_budgets = await _mc_store.load_all()
load_recall_budget_cache(_recall_budgets)
logger.info("Memory config cache warmed: %d user override(s)", len(_recall_budgets))
except Exception:
logger.exception("Memory config cache warm-up failed (non-fatal)")
# Ensure admin user exists (auto-create on first boot)
# Must run AFTER langgraph_runtime so app.state.store is available for thread migration
await _ensure_admin_user(app)
@@ -402,6 +507,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill curator scheduler failed to start (non-fatal)")
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
try:
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
except Exception:
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
@@ -439,6 +555,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Curator scheduler shutdown error")
# 停止 AI 写作清理调度器
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
if cleanup_task is not None:
cleanup_task.cancel()
try:
await cleanup_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("AI writing cleanup scheduler shutdown error")
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
# ``langgraph_runtime()`` 的 contextmanager 统一关闭,无需在这里特殊处理。
try:
scheduler_service = getattr(app.state, "scheduled_task_service", None)
if scheduler_service is not None:
@@ -685,6 +816,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Scheduled task API is mounted at /api/scheduled-tasks
app.include_router(scheduled_tasks.router)
# Public, unauthenticated read-only scheduled task API
# (/api/public/scheduled-tasks/*). Lets external systems query tasks and
# surface results without a DeerFlow login.
app.include_router(public_scheduled_tasks.router)
# System settings API is mounted at /api/system-settings
app.include_router(system_settings.router)
@@ -725,6 +861,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Stateless Runs API (stream/wait without a pre-existing thread)
app.include_router(runs.router)
# Multi-agent orchestration (coordinator + sub-agent broadcast/dispatch)
app.include_router(multi_agent.router)
@app.get("/health", tags=["health"])
async def health_check() -> dict:
"""Health check endpoint.
@@ -37,9 +37,17 @@ def should_check_csrf(request: Request) -> bool:
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
return False
path = request.url.path.rstrip("/")
path = request.url.path
stripped = path.rstrip("/")
# Exempt /api/v1/auth/me endpoint
if path == "/api/v1/auth/me":
if stripped == "/api/v1/auth/me":
return False
# Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES).
# These intentionally have no cookie session, so the Double-Submit Cookie
# pattern is both impossible (no token to compare) and unnecessary (no
# session for an attacker to ride). Without this exemption every public
# PUT/POST returns 403 even though auth was bypassed earlier in the chain.
if path.startswith("/api/public/"):
return False
return True
@@ -130,14 +130,24 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.thread_share_store = make_thread_share_store(sf)
from deerflow.persistence.ai_writing_sessions import make_ai_writing_session_store
from deerflow.persistence.ai_writing_sessions import (
make_ai_writing_session_store,
register_default_store as _register_ai_writing_default_store,
)
app.state.ai_writing_session_store = make_ai_writing_session_store(sf)
# 让 LangGraph Server 路径下的 AI 写作节点能跨 app/harness 边界拿到 repo。
# 老 /start /stream /resume 路由仍走 request.app.state,互不影响。
_register_ai_writing_default_store(app.state.ai_writing_session_store)
from deerflow.persistence.article_types import make_article_type_store
app.state.article_type_store = make_article_type_store(sf)
from deerflow.persistence.memory_config import make_user_memory_config_store
app.state.memory_config_store = make_user_memory_config_store(sf)
# Run event store (has its own factory with config-driven backend selection)
run_events_config = getattr(config, "run_events", None)
app.state.run_event_store = make_run_event_store(run_events_config)
@@ -261,6 +271,14 @@ def get_article_type_store(request: Request):
return getattr(request.app.state, "article_type_store", None)
def get_memory_config_store(request: Request):
"""Return the per-user memory config store."""
val = getattr(request.app.state, "memory_config_store", None)
if val is None:
raise HTTPException(status_code=503, detail="Memory config store not available")
return val
def get_run_context(request: Request) -> RunContext:
"""Build a :class:`RunContext` from ``app.state`` singletons.
@@ -19,14 +19,15 @@ from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import (
get_agent_store,
get_checkpointer,
get_current_user_from_request,
get_thread_store,
)
from deerflow.agents.memory import BuiltinFileProvider
from deerflow.config.agents_config import AGENT_NAME_PATTERN
from deerflow.config.paths import get_paths
from deerflow.runtime import serialize_channel_values
from app.gateway.routers.memory import _build_v1_provider, _is_v1
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
@@ -127,52 +128,289 @@ async def list_user_thread_messages(
class AdminMemoryBucket(BaseModel):
name: str = Field(..., description="Bucket name: 'global' or a custom-agent name")
name: str = Field(..., description="Bucket name: 'global' or the agent id (used for CRUD)")
display_name: str = Field("", description="Human-readable name shown in the UI")
data: dict = Field(default_factory=dict, description="Raw memory data for the bucket")
class AdminMemoryEntryAddRequest(BaseModel):
content: str = Field(..., min_length=1, description="Entry content to add")
agent: str | None = Field(None, description="Agent name; None = global USER.md bucket")
class AdminMemoryEntryReplaceRequest(BaseModel):
old_text: str = Field(..., min_length=1, description="Short unique substring to locate the entry")
content: str = Field(..., min_length=1, description="New content to replace with")
agent: str | None = Field(None, description="Agent name; None = global USER.md bucket")
class AdminMemoryResponse(BaseModel):
user_id: str
version: str = Field(default="v2", description="记忆版本:v1 或 v2")
buckets: list[AdminMemoryBucket] = Field(default_factory=list)
def _read_memory_bucket(user_id: str, agent_id: str | None) -> dict:
"""Read one Memory V2 file into a JSON-able ``{"entries": [...]}`` dict.
``agent_id is None`` -> the user's ``USER.md`` (profile, cross-agent);
otherwise that agent's ``MEMORY.md`` (per-agent working notes).
"""
"""Read one Memory V2 file into a JSON-able ``{"entries": [...]}`` dict."""
provider = BuiltinFileProvider()
provider.initialize(user_id=user_id, agent_id=agent_id)
target = "user" if agent_id is None else "memory"
return {"entries": provider.read(target)}
def _read_v1_memory_bucket(user_id: str, agent_id: str | None) -> dict:
"""Read one V1 memory.json into its raw dict (facts / user / history)."""
provider = _build_v1_provider(user_id, agent_id)
return provider.read_v1_data()
@router.get("/{user_id}/memory", response_model=AdminMemoryResponse)
async def get_user_memory(user_id: str, request: Request) -> AdminMemoryResponse:
"""Return every memory bucket for one user.
Memory V2 layout: the ``global`` bucket is ``USER.md`` (user profile);
each per-agent bucket is that agent's ``MEMORY.md`` working notes.
"""
"""Return every memory bucket for one user (V1 or V2 depending on config)."""
await _require_admin(request)
buckets: list[AdminMemoryBucket] = [
AdminMemoryBucket(name="global", data=_read_memory_bucket(user_id, None))
]
# Source agents from the database: only agents owned by this specific user.
agent_store = get_agent_store(request)
all_agents = await agent_store.list_visible(user_id)
owned_agents = sorted(
[a for a in all_agents if a.get("user_id") == user_id],
key=lambda a: (a.get("name") or a["id"]).lower(),
)
agents_root = get_paths().user_dir(user_id) / "agents"
if agents_root.is_dir():
for entry in sorted(agents_root.iterdir(), key=lambda p: p.name.lower()):
if not entry.is_dir() or not (entry / "MEMORY.md").exists():
continue
if not AGENT_NAME_PATTERN.match(entry.name):
continue
if _is_v1(user_id):
# V1 模式:每个桶读 memory.json,内容为 {facts, user, history}
buckets: list[AdminMemoryBucket] = []
try:
buckets.append(AdminMemoryBucket(name="default", display_name="默认助手", data=_read_v1_memory_bucket(user_id, None)))
except Exception:
logger.exception("Failed to read V1 global memory for user %s", user_id)
buckets.append(AdminMemoryBucket(name="default", display_name="默认助手", data={}))
for agent in owned_agents:
agent_id: str = agent["id"]
display_name: str = agent.get("name") or agent_id
try:
data = _read_memory_bucket(user_id, entry.name)
data = _read_v1_memory_bucket(user_id, agent_id)
except Exception:
logger.warning("Failed to read memory bucket %s for user", entry.name)
continue
buckets.append(AdminMemoryBucket(name=entry.name, data=data))
data = {}
buckets.append(AdminMemoryBucket(name=agent_id, display_name=display_name, data=data))
return AdminMemoryResponse(user_id=user_id, buckets=buckets)
return AdminMemoryResponse(user_id=user_id, version="v1", buckets=buckets)
# V2 模式:读 USER.md / MEMORY.md,内容为 {entries: [...]}
buckets = [
AdminMemoryBucket(name="default", display_name="默认助手", data=_read_memory_bucket(user_id, None))
]
for agent in owned_agents:
agent_id = agent["id"]
display_name = agent.get("name") or agent_id
try:
data = _read_memory_bucket(user_id, agent_id)
except Exception:
data = {"entries": []}
buckets.append(AdminMemoryBucket(name=agent_id, display_name=display_name, data=data))
return AdminMemoryResponse(user_id=user_id, version="v2", buckets=buckets)
@router.post("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
async def admin_add_memory_entry(
user_id: str,
body: AdminMemoryEntryAddRequest,
request: Request,
) -> AdminMemoryBucket:
"""Add a memory entry for a user (admin only)."""
await _require_admin(request)
target = "user" if body.agent is None else "memory"
provider = BuiltinFileProvider()
provider.initialize(user_id=user_id, agent_id=body.agent)
try:
result = provider.add(target, body.content)
if result is False:
raise HTTPException(status_code=400, detail="Failed to add entry (quota exceeded or write error)")
except HTTPException:
raise
except Exception as exc:
logger.error("admin_add_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to write memory entry") from exc
bucket_name = "global" if body.agent is None else body.agent
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, body.agent))
@router.put("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
async def admin_replace_memory_entry(
user_id: str,
body: AdminMemoryEntryReplaceRequest,
request: Request,
) -> AdminMemoryBucket:
"""Replace a memory entry for a user (admin only)."""
await _require_admin(request)
target = "user" if body.agent is None else "memory"
provider = BuiltinFileProvider()
provider.initialize(user_id=user_id, agent_id=body.agent)
try:
result = provider.replace(target, body.old_text, body.content)
if result is False:
raise HTTPException(status_code=404, detail="Entry not found or write failed")
except HTTPException:
raise
except Exception as exc:
logger.error("admin_replace_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to replace memory entry") from exc
bucket_name = "global" if body.agent is None else body.agent
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, body.agent))
@router.delete("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
async def admin_delete_memory_entry(
user_id: str,
request: Request,
old_text: str = Query(..., min_length=1, description="Short unique substring to locate the entry"),
agent: str | None = Query(None, description="Agent name; None = global USER.md bucket"),
) -> AdminMemoryBucket:
"""Delete a memory entry for a user (admin only)."""
await _require_admin(request)
target = "user" if agent is None else "memory"
provider = BuiltinFileProvider()
provider.initialize(user_id=user_id, agent_id=agent)
try:
result = provider.remove(target, old_text)
if result is False:
raise HTTPException(status_code=404, detail="Entry not found")
except HTTPException:
raise
except Exception as exc:
logger.error("admin_delete_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to delete memory entry") from exc
bucket_name = "global" if agent is None else agent
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, agent))
# ── V1 Memory CRUD (admin) ─────────────────────────────────────────────────
_V1_USER_FIELDS = {"workContext", "personalContext", "topOfMind"}
_V1_HISTORY_FIELDS = {"recentMonths", "earlierContext", "longTermBackground"}
class AdminV1SectionUpdateRequest(BaseModel):
section_type: str = Field(..., description="'user' 或 'history'")
field: str = Field(..., description="字段名,如 workContext / recentMonths")
summary: str = Field(..., description="新的摘要文本")
agent: str | None = Field(None, description="Agent id;None 表示用户级全局记忆")
class AdminV1FactAddRequest(BaseModel):
content: str = Field(..., min_length=1)
category: str = Field(default="context")
confidence: float = Field(default=0.8, ge=0.0, le=1.0)
agent: str | None = Field(None)
class AdminV1FactUpdateRequest(BaseModel):
content: str | None = Field(None)
category: str | None = Field(None)
confidence: float | None = Field(None, ge=0.0, le=1.0)
agent: str | None = Field(None)
@router.put("/{user_id}/memory/v1/sections", response_model=AdminMemoryBucket)
async def admin_update_v1_section(
user_id: str,
body: AdminV1SectionUpdateRequest,
request: Request,
) -> AdminMemoryBucket:
"""更新 V1 记忆的某个摘要字段(用户画像 / 历史)。"""
await _require_admin(request)
if body.section_type == "user" and body.field not in _V1_USER_FIELDS:
raise HTTPException(status_code=400, detail=f"无效 user 字段:{body.field}")
if body.section_type == "history" and body.field not in _V1_HISTORY_FIELDS:
raise HTTPException(status_code=400, detail=f"无效 history 字段:{body.field}")
if body.section_type not in ("user", "history"):
raise HTTPException(status_code=400, detail="section_type 须为 'user' 或 'history'")
from deerflow.agents.memory.v1.storage import get_memory_storage, utc_now_iso_z
storage = get_memory_storage()
data = storage.load(body.agent, user_id=user_id)
data.setdefault(body.section_type, {})[body.field] = {
"summary": body.summary,
"updatedAt": utc_now_iso_z(),
}
if not storage.save(data, body.agent, user_id=user_id):
raise HTTPException(status_code=500, detail="写入失败")
bucket_name = "global" if body.agent is None else body.agent
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
@router.post("/{user_id}/memory/v1/facts", response_model=AdminMemoryBucket)
async def admin_add_v1_fact(
user_id: str,
body: AdminV1FactAddRequest,
request: Request,
) -> AdminMemoryBucket:
"""新增一条 V1 事实记录。"""
await _require_admin(request)
try:
from deerflow.agents.memory.v1.updater import create_memory_fact
create_memory_fact(
body.content,
category=body.category,
confidence=body.confidence,
agent_name=body.agent,
user_id=user_id,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail="新增失败") from exc
bucket_name = "global" if body.agent is None else body.agent
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
@router.put("/{user_id}/memory/v1/facts/{fact_id}", response_model=AdminMemoryBucket)
async def admin_update_v1_fact(
user_id: str,
fact_id: str,
body: AdminV1FactUpdateRequest,
request: Request,
) -> AdminMemoryBucket:
"""更新一条 V1 事实记录(内容 / 类别 / 置信度)。"""
await _require_admin(request)
try:
from deerflow.agents.memory.v1.updater import update_memory_fact
update_memory_fact(
fact_id,
content=body.content,
category=body.category,
confidence=body.confidence,
agent_name=body.agent,
user_id=user_id,
)
except KeyError:
raise HTTPException(status_code=404, detail=f"事实 '{fact_id}' 不存在")
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail="更新失败") from exc
bucket_name = "global" if body.agent is None else body.agent
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
@router.delete("/{user_id}/memory/v1/facts/{fact_id}", response_model=AdminMemoryBucket)
async def admin_delete_v1_fact(
user_id: str,
fact_id: str,
request: Request,
agent: str | None = Query(None, description="Agent id;None 表示用户级全局记忆"),
) -> AdminMemoryBucket:
"""删除一条 V1 事实记录。"""
await _require_admin(request)
try:
from deerflow.agents.memory.v1.updater import delete_memory_fact
delete_memory_fact(fact_id, agent_name=agent, user_id=user_id)
except KeyError:
raise HTTPException(status_code=404, detail=f"事实 '{fact_id}' 不存在")
except Exception as exc:
raise HTTPException(status_code=500, detail="删除失败") from exc
bucket_name = "global" if agent is None else agent
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, agent))
@@ -45,6 +45,11 @@ class AgentResponse(BaseModel):
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
featured_order: int | None = Field(
default=None,
description="Admin-curated homepage rank. Null = not featured. Lower sorts first.",
)
class AgentsListResponse(BaseModel):
"""Response model for listing all visible agents."""
@@ -55,6 +60,10 @@ class AgentsListResponse(BaseModel):
class AgentCreateRequest(BaseModel):
"""Request body for creating a custom agent."""
id: str | None = Field(
default=None,
description="Optional explicit storage id. Must match the storage id pattern. If omitted, a UUID is generated.",
)
name: str = Field(..., description="Display name; Chinese and duplicate names are allowed")
description: str = Field(default="", description="Agent description")
soul: str = Field(default="", description="SOUL.md content - agent personality and behavioral guardrails")
@@ -76,6 +85,28 @@ class AgentUpdateRequest(BaseModel):
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
class AgentSelectorResponse(BaseModel):
"""Response for the homepage agent selector dropdown.
The list is fully curated by admins (``featured_order`` on each row).
Non-admin users get a read-only view of the same list.
"""
agents: list[AgentResponse] = Field(
default_factory=list,
description="Admin-curated agents, sorted by featured_order ascending.",
)
class AgentFeaturedRequest(BaseModel):
"""Request body for setting/clearing an agent's homepage featured rank."""
order: int | None = Field(
default=None,
description="Featured rank. Lower sorts first. Pass null to remove the agent from the homepage selector.",
)
def _require_agents_api_enabled() -> None:
"""Reject access unless the custom-agent management API is explicitly enabled."""
if not get_agents_api_config().enabled:
@@ -179,7 +210,45 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
logger.exception("Failed to sync legacy folder-backed agents")
def _record_to_response(record: dict[str, Any], *, include_soul: bool = False) -> AgentResponse:
def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
"""Lazily restore an agent's on-disk artifacts from its DB record.
The DB row is authoritative for the agent's existence. If the matching
``{base_dir}/agents/{id}/`` directory is missing (data dir moved between
machines, filesystem wiped, etc.) we recreate it with a minimal
``config.yaml`` + empty ``SOUL.md`` synthesized from the row. This restores
a usable agent without requiring the admin to manually delete + recreate.
"""
agent_id = record.get("id")
if not agent_id:
return
try:
agent_dir = get_paths().agent_dir(agent_id)
if agent_dir.exists():
return
logger.warning(
"Agent dir missing for '%s' — rebuilding minimal config.yaml + SOUL.md from DB",
agent_id,
)
_write_agent_config(
agent_id,
name=record.get("name") or agent_id,
description=record.get("description") or "",
)
_write_agent_soul(agent_id, "")
except Exception:
# Self-heal is best-effort. ``load_agent_config`` already tolerates a
# missing dir, so the worst case is the agent runs with default
# settings until someone re-edits it.
logger.exception("Failed to self-heal agent dir for '%s'", agent_id)
def _record_to_response(
record: dict[str, Any],
*,
include_soul: bool = False,
) -> AgentResponse:
_ensure_agent_dir_from_record(record)
soul: str | None = None
if include_soul:
soul = load_agent_soul(record["id"]) or ""
@@ -199,6 +268,7 @@ def _record_to_response(record: dict[str, Any], *, include_soul: bool = False) -
model=extras["model"],
tool_groups=extras["tool_groups"],
skills=extras["skills"],
featured_order=record.get("featured_order"),
)
@@ -284,6 +354,32 @@ async def check_agent_name(name: str) -> dict[str, Any]:
return {"available": True, "name": normalized}
@router.get(
"/agents/selector",
response_model=AgentSelectorResponse,
summary="Agent Homepage Selector",
description=(
"Return the admin-curated agent list shown on the new-chat homepage. "
"Sorted by ``featured_order`` ascending. Visibility-filtered, so an "
"agent the caller can't see (author un-published) is silently omitted."
),
)
async def get_agent_selector(request: Request) -> AgentSelectorResponse:
_require_agents_api_enabled()
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
user_id = _current_user_id(request)
records = await store.list_featured(user_id)
return AgentSelectorResponse(agents=[_record_to_response(r) for r in records])
except HTTPException:
raise
except Exception as e:
logger.error("Failed to load agent selector: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to load agent selector: {str(e)}") from e
@router.get(
"/agents/{agent_id}",
response_model=AgentResponse,
@@ -297,7 +393,8 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
record = await store.get_visible(agent_id, _current_user_id(request))
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
return _record_to_response(record, include_soul=True)
@@ -318,13 +415,19 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
async def create_agent_endpoint(request: AgentCreateRequest, http_request: Request) -> AgentResponse:
_require_agents_api_enabled()
name = _validate_display_name(request.name)
agent_id = uuid4().hex
store = get_agent_store(http_request)
if request.id:
agent_id = _validate_storage_id(request.id)
if await store.get_any(agent_id) is not None:
raise HTTPException(status_code=409, detail=f"Agent id '{agent_id}' already exists")
else:
agent_id = uuid4().hex
agent_dir = get_paths().agent_dir(agent_id)
try:
store = get_agent_store(http_request)
if agent_dir.exists():
raise HTTPException(status_code=409, detail=f"Generated agent id '{agent_id}' already exists")
raise HTTPException(status_code=409, detail=f"Agent id '{agent_id}' already exists")
_write_agent_config(
agent_id,
@@ -514,3 +617,91 @@ async def delete_agent(agent_id: str, request: Request) -> None:
except Exception as e:
logger.error("Failed to delete agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to delete agent: {str(e)}") from e
@router.put(
"/agents/{agent_id}/featured",
response_model=AgentResponse,
summary="Set Agent Featured Rank (admin)",
description=(
"Admin-only. Set or clear the homepage-selector featured rank for any agent. "
"Requires the agent to be ``published=true`` when a non-null rank is supplied; "
"passing ``order: null`` removes the agent from the selector and is always allowed."
),
)
async def set_agent_featured(
agent_id: str,
body: AgentFeaturedRequest,
request: Request,
) -> AgentResponse:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
if not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can set the homepage featured rank")
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
record = await store.get_any(agent_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
if body.order is not None and not record.get("published"):
raise HTTPException(status_code=409, detail="Only published agents can be featured on the homepage")
updated = await store.set_featured(agent_id, body.order)
if updated is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
logger.info("Set agent '%s' featured_order=%s by admin", agent_id, body.order)
return _record_to_response(updated, include_soul=True)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to set featured for agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
class AgentFeaturedReorderRequest(BaseModel):
"""Request body for admin to bulk-reorder the homepage featured list."""
agent_ids: list[str] = Field(
default_factory=list,
description=(
"Featured agent ids in the desired top→bottom display order. Ids not "
"currently featured are silently skipped, so the client can send the "
"current visible order without first reconciling with the server."
),
)
@router.put(
"/agents/featured/order",
status_code=204,
summary="Reorder Featured Agents (admin)",
description=(
"Admin-only. Bulk-assign ``featured_order`` to each id in the order received "
"(``0`` is topmost). After this, the homepage selector renders agents in "
"exactly the supplied order. Ids not already featured are ignored."
),
)
async def reorder_featured_agents(
body: AgentFeaturedReorderRequest,
request: Request,
) -> None:
_require_agents_api_enabled()
if not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can reorder the homepage selector")
try:
store = get_agent_store(request)
for index, agent_id in enumerate(body.agent_ids):
# ``set_featured`` is a no-op for unknown ids — it returns None and
# we move on without raising, matching the docstring.
await store.set_featured(agent_id, index)
logger.info("Admin reordered featured agents (%d ids)", len(body.agent_ids))
except HTTPException:
raise
except Exception as e:
logger.error("Failed to reorder featured agents: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to reorder featured agents: {str(e)}") from e
@@ -1,31 +1,26 @@
"""AI 写作路由 —— 独立于原有路由,前缀 /api/ai-writing/
"""AI 写作路由 —— 业务表 CRUD + transcript 持久化 + admin cleanup。
写作主流程(``/start /stream /resume /state``)已迁到 LangGraph Server 的
``ai_writing`` assistant(前端走 ``useAIWritingStream`` hook + SDK),路由层
不再承载 graph 推进,只保留以下「不依赖 graph 状态」的辅助接口:
POST /api/ai-writing/start 启动写作流程,返回 session_id
GET /api/ai-writing/{id}/stream SSE 流(进度/干预/草稿事件)
POST /api/ai-writing/{id}/resume 用户干预响应,恢复流程
GET /api/ai-writing/{id}/state 查询当前流程状态
GET /api/ai-writing/sessions 列出用户的会话历史
GET /api/ai-writing/sessions/{id} 获取单个历史会话
PATCH /api/ai-writing/sessions/{id} 重命名会话标题
DELETE /api/ai-writing/sessions/{id} 删除会话
PUT /api/ai-writing/sessions/{id}/transcript 保存完整对话时间线
GET /api/ai-writing/article-types 列出所有文章类型
POST /api/ai-writing/article-types 新增文章类型
PATCH /api/ai-writing/article-types/{id} 更新文章类型
DELETE /api/ai-writing/article-types/{id} 删除文章类型
POST /api/ai-writing/admin/cleanup (admin)手动清理会话
"""
from __future__ import annotations
import asyncio
import json
import logging
import uuid
from collections.abc import AsyncGenerator
from datetime import UTC, datetime
from typing import Optional
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import StreamingResponse
from langgraph.types import Command
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
@@ -34,35 +29,32 @@ router = APIRouter(prefix="/api/ai-writing", tags=["ai-writing"])
# ── 请求/响应模型 ─────────────────────────────────────────────────────────────
class StartWritingRequest(BaseModel):
user_intent: str
article_type: str = "analysis"
target_audience: str = "普通读者"
word_count_target: int = 800
model_name: str = ""
class StartWritingResponse(BaseModel):
session_id: str
message: str = "写作流程已启动"
class ResumeRequest(BaseModel):
action: str
pause_point: str = ""
approved_material_ids: list[str] = Field(default_factory=list)
extra_keywords: list[str] = Field(default_factory=list)
edited_outline: Optional[dict] = None
outline_feedback: str = ""
user_revision_notes: str = ""
override_verdict: bool = False
selected_issue_indices: Optional[list[int]] = None
class RenameSessionRequest(BaseModel):
title: str
class AdminCleanupRequest(BaseModel):
"""手动清理接口的可选覆盖参数。
全部留空 → 用 config.yaml 的 ai_writing.cleanup 段默认值;显式传值
→ 覆盖配置(一般用于运维想验证一个临时阈值,例如清掉「最近 1 小时
以前的」会话来模拟整轮调度)。
"""
retention_days: Optional[int] = Field(default=None, ge=0, description="保留多少天以内的会话;0 = 不保留")
only_finished: Optional[bool] = Field(default=None, description="True 时只清 done/error 状态")
batch_size: Optional[int] = Field(default=None, ge=1, le=10000, description="单批最多删多少行")
class AdminCleanupResponse(BaseModel):
deleted_db: int
deleted_checkpoints: int
# 因「仍在跑 / 等用户干预」被保护掉的会话数(threads_meta.status in
# running/interrupted)。即使 retention_days=0 也不会秒杀用户正在写的稿。
skipped_active: int = 0
elapsed_seconds: float
params: dict
class SaveTranscriptRequest(BaseModel):
# 前端组装的完整对话时间线(progressEvents + completedInterventions)
transcript: dict
@@ -113,140 +105,7 @@ class SessionResponse(BaseModel):
updated_at: str
# ── 内部状态 ──────────────────────────────────────────────────────────────────
# session_id → asyncio.Queue
_event_queues: dict[str, asyncio.Queue] = {}
# session_id → model_name(resume 时复用)
_session_model_names: dict[str, str] = {}
# session_id → AIWritingSessionRepository(for DB saves)
_session_repos: dict[str, object] = {}
# session_id → user_id (str | None, already resolved)
_session_user_ids: dict[str, str | None] = {}
# session_id → accumulated intervention records
_session_interventions: dict[str, list] = {}
# session_id → True if session ended with error
_session_error_flags: dict[str, bool] = {}
# session_id → valid_article_types(list[dict] with key/label)
_session_article_types: dict[str, list[dict]] = {}
# session_id → pending await_user event tuple for SSE reconnect re-delivery
# asyncio.Queue is consume-once; if the old SSE generator dequeued the await_user event
# before the client disconnected, the event is lost from the queue. We save a copy here
# so that a new SSE connection (e.g., after user clicks "继续") can re-inject it.
_session_pending_pause: dict[str, tuple] = {}
# ── 工具函数 ──────────────────────────────────────────────────────────────────
def _sse(event: str, data: dict) -> str:
return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
def _ai_writing_config() -> dict:
"""读取 config.yaml 的 ai_writing 段(含素材检索 provider 配置)。"""
try:
from deerflow.config import get_app_config
aw = getattr(get_app_config(), "ai_writing", None)
if isinstance(aw, dict):
return aw
return dict(aw) if aw else {}
except Exception:
return {}
def _make_config(session_id: str, model_name: str = "", valid_article_types: list[dict] | None = None) -> dict:
return {
"configurable": {
"thread_id": session_id,
"model_name": model_name or _session_model_names.get(session_id, ""),
"ai_writing_config": _ai_writing_config(),
# 把 SSE 队列注入,供草稿/大纲节点直接推流式 chunk
"sse_queue": _event_queues.get(session_id),
"sse_session_id": session_id,
# 动态文章类型列表,intent_parser 从此处读取有效枚举
"valid_article_types": valid_article_types or _session_article_types.get(session_id),
}
}
def _truncate_title(text: str, max_len: int = 50) -> str:
text = text.strip()
return text[:max_len] + ("…" if len(text) > max_len else "")
# ── 路由 ──────────────────────────────────────────────────────────────────────
@router.post("/start", response_model=StartWritingResponse)
async def start_writing(body: StartWritingRequest, request: Request):
"""启动写作流程,返回 session_id。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
from app.gateway.deps import get_ai_writing_session_store, get_article_type_store, get_current_user
session_id = str(uuid.uuid4())
queue: asyncio.Queue = asyncio.Queue()
_event_queues[session_id] = queue
_session_model_names[session_id] = body.model_name
_session_interventions[session_id] = []
# 加载动态文章类型列表,注入给 intent_parser 节点
article_type_store = get_article_type_store(request)
valid_article_types: list[dict] | None = None
if article_type_store is not None:
try:
records = await article_type_store.list()
if records:
valid_article_types = [{"key": r["key"], "label": r["label"]} for r in records]
except Exception:
logger.exception("ai_writing: failed to load article types for session=%s", session_id)
_session_article_types[session_id] = valid_article_types or []
# 持久化会话记录(DB 可能未配置,则跳过)
repo = get_ai_writing_session_store(request)
user_id = await get_current_user(request)
if repo is not None:
try:
title = _truncate_title(body.user_intent)
await repo.create(
id=session_id,
title=title,
user_intent=body.user_intent,
user_id=user_id,
)
_session_repos[session_id] = repo
_session_user_ids[session_id] = user_id
except Exception:
logger.exception("ai_writing: failed to create session record: session=%s", session_id)
initial_state = {
"user_intent": body.user_intent,
"article_type": body.article_type,
"target_audience": body.target_audience,
"word_count_target": body.word_count_target,
"revision_count": 0,
"max_revisions": 3,
"status": "researching",
"drafts": [],
"progress_events": [],
"material_package": None,
"current_outline": None,
"current_draft": None,
"review_result": None,
"last_user_intervention": None,
}
config = _make_config(session_id, body.model_name, valid_article_types=valid_article_types)
asyncio.create_task(_stream_graph_segment(session_id, initial_state, config, AI_WRITING_GRAPH))
return StartWritingResponse(session_id=session_id)
@router.get("/sessions", response_model=list[SessionResponse])
async def list_sessions(request: Request):
"""列出当前用户的 AI 写作历史会话。"""
@@ -306,19 +165,27 @@ async def delete_session(session_id: str, request: Request):
@router.put("/sessions/{session_id}/transcript")
async def save_transcript(session_id: str, body: SaveTranscriptRequest, request: Request):
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。"""
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。
时间线持久化属于「锦上添花」功能:即便保存失败(如内网部署缺少 transcript
列、或 MySQL TEXT 列超长),也绝不能阻断写作流程。任何异常都吞掉并返回
success=False,由前端忽略即可。
"""
from app.gateway.deps import get_ai_writing_session_store, get_current_user
repo = get_ai_writing_session_store(request)
if repo is None:
return {"success": False}
user_id = await get_current_user(request)
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
return {"success": False, "reason": "store_unavailable"}
try:
user_id = await get_current_user(request)
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
except Exception:
logger.exception("ai_writing: failed to save transcript: session=%s", session_id)
return {"success": False, "reason": "save_failed"}
if updated is None:
raise HTTPException(status_code=404, detail="Session not found")
# 会话不存在或不属于当前用户:同样不抛 500,返回 success=False 即可
return {"success": False, "reason": "session_not_found"}
return {"success": True}
@router.get("/article-types", response_model=list[ArticleTypeResponse])
async def list_article_types(request: Request):
"""列出所有文章类型,按 sort_order 升序排列。"""
@@ -392,249 +259,109 @@ async def delete_article_type(article_type_id: str, request: Request):
return {"success": True}
@router.get("/{session_id}/stream")
async def stream_progress(session_id: str):
"""SSE 流:推送进度事件、干预请求、草稿内容。"""
return StreamingResponse(
_sse_generator(session_id),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
# ── 管理接口 ──────────────────────────────────────────────────────────────────
@router.post("/{session_id}/resume")
async def resume_writing(session_id: str, body: ResumeRequest, request: Request):
"""用户提交干预决策,恢复暂停的流程。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
from app.gateway.deps import get_current_user
async def _require_admin(request: Request) -> None:
"""放行 admin / auth-disabled,其余 403。
if session_id not in _event_queues:
raise HTTPException(status_code=404, detail="Session not found")
config = _make_config(session_id)
intervention = {
"action": body.action,
"approvedMaterialIds": body.approved_material_ids,
"extraKeywords": body.extra_keywords,
"editedOutline": body.edited_outline,
"outlineFeedback": body.outline_feedback,
"userRevisionNotes": body.user_revision_notes,
"overrideVerdict": body.override_verdict,
"selectedIssueIndices": body.selected_issue_indices,
}
# Record this intervention for session history
record = {
"pausePoint": body.pause_point,
"action": body.action,
"timestamp": datetime.now(UTC).isoformat(),
}
if session_id in _session_interventions:
_session_interventions[session_id].append(record)
repo = _session_repos.get(session_id)
user_id = _session_user_ids.get(session_id)
if repo is not None:
try:
await repo.update(
session_id,
user_id=user_id,
completed_interventions=_session_interventions[session_id],
)
except Exception:
logger.exception("ai_writing: failed to save intervention: session=%s", session_id)
# Clear pending pause so SSE reconnect won't re-inject the stale await_user event
_session_pending_pause.pop(session_id, None)
asyncio.create_task(
_stream_graph_segment(session_id, Command(resume=intervention), config, AI_WRITING_GRAPH)
)
return {"message": "已提交,流程恢复中"}
@router.get("/{session_id}/state")
async def get_state(session_id: str):
"""查询当前流程状态快照。"""
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
config = _make_config(session_id)
snapshot = await AI_WRITING_GRAPH.aget_state(config)
if not snapshot or not snapshot.values:
raise HTTPException(status_code=404, detail="Session not found")
return snapshot.values
# ── 内部辅助 ──────────────────────────────────────────────────────────────────
async def _stream_graph_segment(
session_id: str,
input_or_command,
config: dict,
graph,
):
"""运行一个 Graph 片段(初始运行或 resume),将输出推入队列。
遇到 interrupt 时停止(不关闭队列),等待下次 resume。
Graph 结束时关闭队列(放入 None 哨兵)。
与 scheduled_tasks._require_admin 同套语义:auth_disabled 模式下
get_optional_user_from_request 返回 None,此时放行(开发环境一致体验)。
"""
queue = _event_queues.get(session_id)
if queue is None:
return
from app.gateway.deps import get_optional_user_from_request
reached_end = False
is_error = False
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="AI 写作清理管理接口仅限管理员")
def _resolve_cleanup_params(override: AdminCleanupRequest) -> dict:
"""合并 config.yaml 默认值 + 请求显式覆盖。
显式传 0 / False 也算覆盖(用 ``is not None`` 判定),方便测「保留 0 天 =
全删」这种极端场景。
"""
from deerflow.config import get_app_config
raw = getattr(get_app_config(), "ai_writing", None) or {}
cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
return {
"retention_days": (
override.retention_days if override.retention_days is not None
else int(cfg.get("retention_days", 7))
),
"only_finished": (
override.only_finished if override.only_finished is not None
else bool(cfg.get("delete_only_finished", False))
),
"batch_size": (
override.batch_size if override.batch_size is not None
else int(cfg.get("batch_size", 500))
),
}
@router.post("/admin/cleanup", response_model=AdminCleanupResponse)
async def admin_cleanup(
body: AdminCleanupRequest,
request: Request,
) -> AdminCleanupResponse:
"""**仅管理员** 手动触发一次会话清理(不等凌晨 3 点定时调度)。
用途:
- 验证 cleanup 配置改动是否生效
- 内网紧急清理(磁盘满 / MySQL 行数过多)
- 调试:传 ``retention_days: 0`` 一刷拉清空,方便重新跑回归
所有参数留空 → 沿用 config.yaml 的 ``ai_writing.cleanup`` 段默认。显式
传值会覆盖配置但**不修改 config.yaml**(一次性参数,下次定时调度还是用
配置文件的值)。
"""
import time
from app.gateway.ai_writing_cleanup import run_cleanup_once
await _require_admin(request)
params = _resolve_cleanup_params(body)
# 透传主 LangGraph checkpointer / thread_store —— 新路径下 ai_writing 的
# thread 状态在主 checkpointer 里,thread_store 用来按 status 过滤活跃 thread。
# getattr 嵌套防御:测试用 stub Request 时 ``request.app`` 可能不存在。
_app_state = getattr(getattr(request, "app", None), "state", None)
main_checkpointer = getattr(_app_state, "checkpointer", None)
main_thread_store = getattr(_app_state, "thread_store", None)
started = time.monotonic()
try:
async for chunk in graph.astream(input_or_command, config=config, stream_mode="updates"):
node_name = next(iter(chunk))
node_output = chunk[node_name]
# interrupt() 触发
if node_name == "__interrupt__":
# node_output 是 (Interrupt(...),) 元组
interrupt_list = node_output if isinstance(node_output, (list, tuple)) else [node_output]
for intr in interrupt_list:
interrupt_data = intr.value if hasattr(intr, "value") else intr
event_tuple = ("await_user", interrupt_data)
await queue.put(event_tuple)
# Save for SSE reconnect re-delivery: if the old generator consumed this
# event before the client disconnected, a new connection would block forever.
_session_pending_pause[session_id] = event_tuple
return # 等待 resume,不关闭队列
if not isinstance(node_output, dict):
continue
# 推送 progress_events
for evt in node_output.get("progress_events") or []:
await queue.put((_event_type(evt), {**evt, "session_id": session_id}))
# 素材就绪
if pkg := node_output.get("material_package"):
await queue.put(("materials_ready", {
"session_id": session_id,
"keywords": pkg.get("keywords", []),
"materials": pkg.get("materials", []),
"summary": pkg.get("summary", ""),
}))
# 大纲就绪
if outline := node_output.get("current_outline"):
await queue.put(("outline_ready", {"session_id": session_id, "outline": outline}))
# 草稿就绪 — 保存到 DB
if draft := node_output.get("current_draft"):
draft_title = draft.get("title", "")
draft_markdown = draft.get("full_markdown", "")
await queue.put(("draft_ready", {
"session_id": session_id,
"title": draft_title,
"markdown": draft_markdown,
"revision_count": draft.get("revision_count", 0),
}))
repo = _session_repos.get(session_id)
user_id = _session_user_ids.get(session_id)
if repo is not None:
try:
await repo.update(
session_id,
user_id=user_id,
status="draft_ready",
title=draft_title or _truncate_title(
next(iter(_session_interventions.get(session_id, [{}])), {}).get("action", "草稿"),
),
draft_title=draft_title,
draft_markdown=draft_markdown,
)
except Exception:
logger.exception("ai_writing: failed to save draft: session=%s", session_id)
# 编辑审核就绪 — 保存到 DB
if review := node_output.get("review_result"):
await queue.put(("review_ready", {"session_id": session_id, "review_result": review}))
repo = _session_repos.get(session_id)
user_id = _session_user_ids.get(session_id)
if repo is not None:
try:
await repo.update(
session_id,
user_id=user_id,
status="review_ready",
review_result=review,
)
except Exception:
logger.exception("ai_writing: failed to save review: session=%s", session_id)
# 状态 done:Graph 结束
if node_output.get("status") == "done":
reached_end = True
# astream 正常结束 = Graph 执行到 END
reached_end = True
await queue.put(("done", {"session_id": session_id, "message": "写作完成"}))
result = await run_cleanup_once(
**params,
checkpointer=main_checkpointer,
thread_store=main_thread_store,
)
except Exception as e:
logger.exception("ai_writing graph segment failed: session=%s", session_id)
await queue.put(("error", {"session_id": session_id, "message": str(e)}))
is_error = True
reached_end = True
finally:
if reached_end:
# 更新最终状态到 DB
repo = _session_repos.get(session_id)
user_id = _session_user_ids.get(session_id)
if repo is not None:
try:
await repo.update(
session_id,
user_id=user_id,
status="error" if is_error else "done",
)
except Exception:
logger.exception("ai_writing: failed to update final status: session=%s", session_id)
logger.exception("ai_writing admin cleanup failed: params=%s", params)
# 抛 500 是合理的 —— 管理员触发清理失败需要看到失败原因,跟「业务
# 接口被失败拖累」是两回事
raise HTTPException(
status_code=500,
detail=f"清理执行失败: {type(e).__name__}: {e}",
) from e
await queue.put(None) # 哨兵,通知 SSE 生成器关闭
_event_queues.pop(session_id, None)
_session_model_names.pop(session_id, None)
_session_repos.pop(session_id, None)
_session_user_ids.pop(session_id, None)
_session_interventions.pop(session_id, None)
_session_error_flags.pop(session_id, None)
_session_article_types.pop(session_id, None)
_session_pending_pause.pop(session_id, None)
def _event_type(evt: dict) -> str:
t = evt.get("type", "progress")
# 过滤掉已在上方单独推送的事件类型,避免重复
if t in ("materials_ready", "outline_ready", "draft_ready", "review_ready", "await_user",
"draft_chunk", "outline_chunk"):
return "step_done"
return t
async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
"""从队列消费事件,推送为 SSE 帧。"""
# 等待队列创建(最多 5 秒)
for _ in range(50):
if session_id in _event_queues:
break
await asyncio.sleep(0.1)
else:
yield _sse("error", {"message": "Session not found"})
return
queue = _event_queues[session_id]
yield _sse("connected", {"session_id": session_id})
# Re-inject the saved await_user event if the queue is empty and a pending pause exists.
# This covers the case where the old SSE generator dequeued the await_user event and
# yielded it to a disconnecting client — the event is then lost from the consume-once queue.
if queue.empty() and session_id in _session_pending_pause:
await queue.put(_session_pending_pause[session_id])
while True:
item = await queue.get()
if item is None:
break
event_type, data = item
yield _sse(event_type, data)
await asyncio.sleep(0)
elapsed = time.monotonic() - started
logger.info(
"ai_writing admin cleanup done: params=%s deleted_db=%d deleted_cp=%d elapsed=%.2fs",
params,
result["deleted_db"],
result["deleted_checkpoints"],
elapsed,
)
return AdminCleanupResponse(
deleted_db=result["deleted_db"],
deleted_checkpoints=result["deleted_checkpoints"],
skipped_active=result.get("skipped_active", 0),
elapsed_seconds=round(elapsed, 3),
params=params,
)
@@ -21,15 +21,16 @@ from __future__ import annotations
from typing import Any
from fastapi import APIRouter, HTTPException, Query
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_memory_config_store
from deerflow.config.agents_config import AGENT_NAME_PATTERN
from deerflow.config.memory_config import (
USER_WRITABLE_FIELDS,
delete_user_memory_override,
delete_user_memory_override_db,
get_effective_memory_config,
save_user_memory_override,
save_user_memory_override_db,
)
from deerflow.config.paths import get_paths
from deerflow.runtime.user_context import get_effective_user_id
@@ -38,7 +39,7 @@ router = APIRouter(prefix="/api", tags=["memory"])
# 表示"全局桶 (无 agent)"的 agent 取值。
_GLOBAL_AGENT_VALUES: frozenset[str] = frozenset({"", "global", "__global__"})
_GLOBAL_AGENT_VALUES: frozenset[str] = frozenset({"", "global", "__global__", "default"})
def _resolve_agent_name(agent: str | None) -> str | None:
@@ -72,20 +73,38 @@ _AgentQuery = Query(
# ===========================================================================
class MemoryV2DataResponse(BaseModel):
"""Memory V2 builtin provider 的数据视图。"""
class MemoryFact(BaseModel):
"""V1 记忆中的单条事实。"""
user_entries: list[str] = Field(default_factory=list, description="USER.md 条目 (用户画像)")
memory_entries: list[str] = Field(default_factory=list, description="MEMORY.md 条目 (agent 工作笔记)")
user_usage: str = Field(default="", description="USER.md 字符用量,如 '12% — 165/1375 字符'")
memory_usage: str = Field(default="", description="MEMORY.md 字符用量")
id: str = Field(default="", description="事实 UUID")
content: str = Field(default="", description="事实文本")
category: str = Field(default="knowledge", description="事实类别")
confidence: float = Field(default=0.0, description="置信度 0-1")
createdAt: str = Field(default="", description="创建时间 ISO 格式")
sourceError: str = Field(default="", description="纠错类事实:被纠正的原错误描述")
class MemoryV2DataResponse(BaseModel):
"""Memory 数据视图 —— V1/V2 共用,多余字段各版本置空。"""
# V2 字段
user_entries: list[str] = Field(default_factory=list, description="V2: USER.md 条目 (用户画像)")
memory_entries: list[str] = Field(default_factory=list, description="V2: MEMORY.md 条目 (agent 工作笔记)")
user_usage: str = Field(default="", description="字符/计数用量描述")
memory_usage: str = Field(default="", description="字符/计数用量描述")
# V1 字段
facts: list[MemoryFact] = Field(default_factory=list, description="V1: facts 列表 (含置信度)")
user_context: dict[str, str] = Field(default_factory=dict, description="V1: userContext 摘要")
history: dict[str, str] = Field(default_factory=dict, description="V1: history 历史摘要")
class MemoryV2ConfigResponse(BaseModel):
"""Memory V2 生效配置 (含三层覆盖合并结果) + 用户可改字段白名单。"""
"""Memory 生效配置 (含三层覆盖合并结果) + 用户可改字段白名单 + 版本号。"""
config: dict[str, Any] = Field(..., description="按 per-user 覆盖合并后的生效配置")
writable_fields: list[str] = Field(..., description="用户可在运行时覆盖的字段 (点号路径)")
version: str = Field(default="v2", description="当前记忆版本:v1 或 v2")
class MemoryConfigPatchRequest(BaseModel):
@@ -133,6 +152,33 @@ class MemorySearchResponse(BaseModel):
# ===========================================================================
def _is_v1(user_id: str) -> bool:
"""判断当前用户是否处于 V1 记忆模式。"""
config = get_effective_memory_config(user_id)
return getattr(config, "version", "v2") == "v1"
def _build_v1_provider(user_id: str, agent_name: str | None = None):
"""构建并初始化 V1BuiltinProvider。"""
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
config = get_effective_memory_config(user_id)
v1_cfg = config.v1
provider = V1BuiltinProvider(
max_facts=v1_cfg.max_facts,
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
debounce_seconds=v1_cfg.debounce_seconds,
model_name=v1_cfg.model_name,
token_budget=v1_cfg.token_budget,
)
provider.initialize(
user_id=user_id,
agent_id=agent_name,
base_dir=str(get_paths().base_dir),
)
return provider
def _build_builtin_provider(user_id: str, agent_name: str | None):
"""按生效配置构建并初始化一个 V2 builtin provider。"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
@@ -160,7 +206,7 @@ def _validate_v2_target(target: str) -> str:
def _v2_data_response(provider) -> MemoryV2DataResponse:
"""从 builtin provider 读出当前数据,组装成 MemoryV2DataResponse。"""
"""从 V2 builtin provider 读出当前数据。"""
return MemoryV2DataResponse(
user_entries=provider.read("user"),
memory_entries=provider.read("memory"),
@@ -169,6 +215,49 @@ def _v2_data_response(provider) -> MemoryV2DataResponse:
)
def _v1_data_response(provider) -> MemoryV2DataResponse:
"""从 V1 provider 读出当前数据,转换为统一响应格式。
支持新格式 {user: {workContext: {summary, updatedAt}}} 和老格式 {userContext: {workContext: str}}。
"""
data = provider.read_v1_data()
raw_facts = data.get("facts", [])
facts = [
MemoryFact(
id=str(f.get("id", "")),
content=str(f.get("content", "")),
category=str(f.get("category", "knowledge")),
confidence=float(f.get("confidence", 0)),
createdAt=str(f.get("createdAt", "")),
sourceError=str(f.get("sourceError", "")),
)
for f in raw_facts
]
def _extract_section(section: dict) -> dict[str, str]:
"""从 {key: {summary: str, updatedAt: str}} 或 {key: str} 中提取摘要字符串。"""
result = {}
for k, v in section.items():
if isinstance(v, dict):
summary = v.get("summary", "")
else:
summary = str(v)
if summary:
result[k] = summary
return result
# 新格式用 "user" key,老格式用 "userContext" key(已由 storage 迁移,此处兜底)
ctx_raw = data.get("user") or data.get("userContext") or {}
history_raw = data.get("history") or {}
return MemoryV2DataResponse(
facts=sorted(facts, key=lambda f: f.confidence, reverse=True),
user_context=_extract_section(ctx_raw),
history=_extract_section(history_raw),
memory_usage=provider.usage_text(),
)
def _raise_if_write_failed(result: dict) -> None:
"""builtin provider 写操作失败时抛 400。"""
if not result.get("success"):
@@ -183,33 +272,43 @@ def _raise_if_write_failed(result: dict) -> None:
@router.get(
"/memory/v2",
response_model=MemoryV2DataResponse,
summary="Get Memory V2 Data",
description="读取 Memory V2 builtin provider 的 USER.md / MEMORY.md 条目。",
summary="Get Memory Data",
description="读取记忆数据。V1 模式返回 facts 列表;V2 模式返回 USER.md / MEMORY.md 条目。",
)
async def get_memory_v2(agent: str | None = _AgentQuery) -> MemoryV2DataResponse:
"""读取当前用户的 Memory V2 本地记忆。"""
"""读取当前用户的本地记忆。"""
agent_name = _resolve_agent_name(agent)
user_id = get_effective_user_id()
try:
if _is_v1(user_id):
provider = _build_v1_provider(user_id, agent_name)
return _v1_data_response(provider)
provider = _build_builtin_provider(user_id, agent_name)
return _v2_data_response(provider)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail="Failed to read memory V2 data.") from exc
raise HTTPException(status_code=500, detail="Failed to read memory data.") from exc
def _assert_not_v1(user_id: str) -> None:
"""V1 模式下写入操作返回 400。"""
if _is_v1(user_id):
raise HTTPException(status_code=400, detail="V1 模式下记忆由系统自动提取,不支持手动修改。")
@router.post(
"/memory/v2/entries",
response_model=MemoryV2DataResponse,
summary="Add Memory V2 Entry",
description="手动往 USER.md / MEMORY.md 新增一条记忆条目。",
description="手动往 USER.md / MEMORY.md 新增一条记忆条目 (仅 V2 模式)。",
)
async def add_memory_v2_entry(
request: MemoryEntryCreateRequest,
agent: str | None = _AgentQuery,
) -> MemoryV2DataResponse:
"""手动新增一条 V2 记忆条目。"""
_validate_v2_target(request.target)
user_id = get_effective_user_id()
_assert_not_v1(user_id)
_validate_v2_target(request.target)
try:
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
result = provider.add(request.target, request.content)
@@ -225,15 +324,16 @@ async def add_memory_v2_entry(
"/memory/v2/entries",
response_model=MemoryV2DataResponse,
summary="Replace Memory V2 Entry",
description="用 old_text 定位一条记忆条目并替换为新内容。",
description="用 old_text 定位一条记忆条目并替换为新内容 (仅 V2 模式)。",
)
async def replace_memory_v2_entry(
request: MemoryEntryReplaceRequest,
agent: str | None = _AgentQuery,
) -> MemoryV2DataResponse:
"""手动替换一条 V2 记忆条目。"""
_validate_v2_target(request.target)
user_id = get_effective_user_id()
_assert_not_v1(user_id)
_validate_v2_target(request.target)
try:
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
result = provider.replace(request.target, request.old_text, request.content)
@@ -249,7 +349,7 @@ async def replace_memory_v2_entry(
"/memory/v2/entries",
response_model=MemoryV2DataResponse,
summary="Delete Memory V2 Entry",
description="用 old_text 定位并删除一条记忆条目。",
description="用 old_text 定位并删除一条记忆条目 (仅 V2 模式)。",
)
async def delete_memory_v2_entry(
target: str = Query(..., description="memory 或 user"),
@@ -257,8 +357,9 @@ async def delete_memory_v2_entry(
agent: str | None = _AgentQuery,
) -> MemoryV2DataResponse:
"""手动删除一条 V2 记忆条目。"""
_validate_v2_target(target)
user_id = get_effective_user_id()
_assert_not_v1(user_id)
_validate_v2_target(target)
try:
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
result = provider.remove(target, old_text)
@@ -288,6 +389,7 @@ async def get_memory_v2_config() -> MemoryV2ConfigResponse:
return MemoryV2ConfigResponse(
config=config.model_dump(),
writable_fields=sorted(USER_WRITABLE_FIELDS),
version=getattr(config, "version", "v2"),
)
@@ -295,20 +397,24 @@ async def get_memory_v2_config() -> MemoryV2ConfigResponse:
"/memory/v2/config",
response_model=MemoryV2ConfigResponse,
summary="Patch Memory V2 Config",
description="把白名单字段写入当前用户的 per-user 配置覆盖。非白名单字段被忽略。",
description="把 hindsight.recall_budget 写入当前用户的 per-user 配置覆盖 (DB 存储)。",
)
async def patch_memory_v2_config(request: MemoryConfigPatchRequest) -> MemoryV2ConfigResponse:
"""更新当前用户的 per-user 记忆配置覆盖。"""
async def patch_memory_v2_config(request: MemoryConfigPatchRequest, req: Request) -> MemoryV2ConfigResponse:
"""更新当前用户的 per-user 记忆配置覆盖 (仅 hindsight.recall_budget)。"""
user_id = get_effective_user_id()
store = get_memory_config_store(req)
try:
save_user_memory_override(user_id, request.overrides)
except OSError as exc:
await save_user_memory_override_db(user_id, request.overrides, store)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail="Failed to save memory config override.") from exc
config = get_effective_memory_config(user_id)
return MemoryV2ConfigResponse(
config=config.model_dump(),
writable_fields=sorted(USER_WRITABLE_FIELDS),
version=getattr(config, "version", "v2"),
)
@@ -317,24 +423,27 @@ async def patch_memory_v2_config(request: MemoryConfigPatchRequest) -> MemoryV2C
response_model=MemoryV2ConfigResponse,
summary="Delete Memory V2 Config Override",
description=(
"撤销当前用户的 per-user 配置覆盖。传 ?field=injection.enabled 只撤销该字段;"
"不传 field 则清空整个 per-user 覆盖。"
"撤销当前用户的 per-user 配置覆盖。传 ?field=hindsight.recall_budget 只撤销该字段;"
"不传 field 则清空全部覆盖。"
),
)
async def delete_memory_v2_config(
req: Request,
field: str | None = Query(None, description="要撤销的点号路径字段;省略则清空全部覆盖"),
) -> MemoryV2ConfigResponse:
"""撤销当前用户的 per-user 记忆配置覆盖。"""
user_id = get_effective_user_id()
store = get_memory_config_store(req)
try:
delete_user_memory_override(user_id, field)
except OSError as exc:
await delete_user_memory_override_db(user_id, field, store)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail="Failed to delete memory config override.") from exc
config = get_effective_memory_config(user_id)
return MemoryV2ConfigResponse(
config=config.model_dump(),
writable_fields=sorted(USER_WRITABLE_FIELDS),
version=getattr(config, "version", "v2"),
)
@@ -0,0 +1,818 @@
"""Multi-agent orchestration endpoints.
Wraps the legacy ``newpython/OUT.py`` demo into the Gateway. Provides:
* ``POST /api/multi-agent/init`` — create a coordinator agent and per-sub-agent
threads. Returns the ``{agent_name: thread_id}`` map the client passes back
on subsequent calls.
* ``POST /api/multi-agent/run/stream`` — drive a *leader* or *special* agent
via SSE. The leader dispatches work to sub-agents through the
``agent_orchestration`` skill; sub-agents execute and broadcast their result.
Internal calls loop back to the same Gateway over HTTP, reusing the caller's
``Authorization`` header — so the demo's hard-coded ``get_token("ji")`` is gone.
The model used by both the coordinator agent and every streamed run is taken
from the request's ``model`` field.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
import time
from typing import Any, AsyncIterator
import httpx
from fastapi import APIRouter, Body, HTTPException, Request
from fastapi.responses import StreamingResponse
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/multi-agent", tags=["multi-agent"])
# ---------------------------------------------------------------------------
# Loopback helpers
# ---------------------------------------------------------------------------
def _loopback_base(request: Request) -> str:
return f"{request.url.scheme}://{request.url.netloc}"
def _auth_headers(request: Request) -> dict[str, str]:
token = request.headers.get("authorization")
if not token:
raise HTTPException(status_code=401, detail="Missing Authorization header")
return {"Authorization": token, "Content-Type": "application/json"}
async def _create_thread(client: httpx.AsyncClient, base: str, headers: dict[str, str]) -> str:
resp = await client.post(f"{base}/api/threads", json={}, headers=headers)
resp.raise_for_status()
return resp.json()["thread_id"]
async def _get_agent(client: httpx.AsyncClient, base: str, headers: dict[str, str], agent_id: str) -> dict[str, Any]:
resp = await client.get(f"{base}/api/agents/{agent_id}", headers=headers)
resp.raise_for_status()
return resp.json()
async def _create_agent(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
*,
name: str,
description: str,
model: str | None,
skills: list[str] | None,
soul: str,
agent_id: str | None = None,
) -> tuple[bool, dict[str, Any] | str]:
payload: dict[str, Any] = {"name": name, "description": description, "soul": soul}
if agent_id is not None:
payload["id"] = agent_id
if model is not None:
payload["model"] = model
if skills is not None:
payload["skills"] = skills
resp = await client.post(f"{base}/api/agents", json=payload, headers=headers)
if resp.status_code == 201:
return True, resp.json()
return False, resp.text
async def _append_thread_message(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
thread_id: str,
content: str,
) -> None:
state_url = f"{base}/api/threads/{thread_id}/state"
get_resp = await client.get(state_url, headers=headers)
if get_resp.status_code != 200:
logger.warning("get state %s failed: %s %s", thread_id, get_resp.status_code, get_resp.text[:200])
return
state = get_resp.json()
values = state.setdefault("values", {})
messages = values.setdefault("messages", [])
if not isinstance(messages, list):
messages = []
values["messages"] = messages
messages.append({"type": "human", "content": content})
post_resp = await client.post(state_url, json=state, headers=headers)
if post_resp.status_code >= 400:
logger.warning("update state %s failed: %s %s", thread_id, post_resp.status_code, post_resp.text[:200])
async def _broadcast(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
message: str,
agent_threads: dict[str, str],
skip: list[str],
) -> None:
targets = [tid for name, tid in agent_threads.items() if name not in skip]
if not targets:
return
await asyncio.gather(
*(_append_thread_message(client, base, headers, tid, message) for tid in targets),
return_exceptions=True,
)
_TOOL_CALL_PATTERN = re.compile(r'\.py\s+"([^"]*)"\s+"([^"]*)"')
# Mirrors the upstream agent-id validator in the LangGraph runtime. Reject early
# so callers get a clear 400 here instead of a stream that aborts with HTTP 422
# from the loopback /api/threads/.../runs/stream endpoint.
_AGENT_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
def _validate_agent_id(value: str, field: str) -> str:
if not _AGENT_ID_PATTERN.match(value):
raise HTTPException(
status_code=400,
detail=(
f"{field} '{value}' is not a valid agent id. "
f"Must match ^[A-Za-z0-9_-]+$ (use the hyphen-case id, not the display name)."
),
)
return value
def _extract_dispatch(tool_call: dict[str, Any]) -> tuple[str, str]:
"""Return ``(agent_name, task)`` parsed from a leader tool call.
Two formats are accepted:
1. **Structured** — ``{"name": "agent_orchestration", "args": {"agent_name": ..., "task": ...}}``.
This is what most LLMs emit when told to "use the agent_orchestration skill",
and the form the roundtable coordinator's SOUL prompt elicits.
2. **Legacy bash argv** — ``{"name": "bash", "args": {"command": '... .py "X" "Y" ...'}}``.
Kept as a fallback so older skills that dispatch through bash still work.
"""
name = (tool_call.get("name") or "").lower()
args = tool_call.get("args")
# Format 1: structured agent_orchestration call.
if name == "agent_orchestration" and isinstance(args, dict):
agent_name = str(args.get("agent_name") or "").strip()
task = str(args.get("task") or "").strip()
if agent_name and task:
return _strip_display_suffix(agent_name), task
# Format 2: legacy bash + .py "X" "Y".
match = _TOOL_CALL_PATTERN.search(str(args))
if match:
return _strip_display_suffix(match.group(1)), match.group(2)
return "", ""
def _strip_display_suffix(agent_name: str) -> str:
"""Tolerate models that paste the human-readable suffix into agent_name.
The coordinator SOUL lists each seat as ``agent_name: roundtable-x(中文):...``
so weaker models sometimes echo the entire ``roundtable-x(中文)`` label
back as the dispatch target. Strip everything from the first non-id
character onward so we recover the bare hyphen-case id; if what's left
still violates the id pattern we hand it back untouched and let the
upstream validator reject it.
"""
head = re.split(r"[^A-Za-z0-9_-]", agent_name, maxsplit=1)[0]
return head or agent_name
def _flatten_content(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, dict):
parts.append(str(item.get("text", "")))
else:
parts.append(str(item))
return "".join(parts)
return str(content)
def _run_payload(
*,
agent_name: str,
model_name: str,
thread_id: str,
new_message: str,
excluded_tools: list[str],
skill_stop_names: list[str],
) -> dict[str, Any]:
return {
"input": {
"messages": [
{
"type": "human",
"content": [{"type": "text", "text": new_message}],
"additional_kwargs": {},
}
]
},
"config": {"recursion_limit": 1000},
"context": {
"agent_name": agent_name,
"model_name": model_name,
"mode": "pro",
"reasoning_effort": "medium",
"thinking_enabled": True,
"is_plan_mode": False,
"subagent_enabled": False,
"thread_id": thread_id,
},
"stream_mode": ["messages-tuple", "values"],
"stream_subgraphs": True,
"stream_resumable": True,
"assistant_id": "lead_agent",
"on_disconnect": "continue",
"excluded_tools": excluded_tools,
"skill_stop_names": skill_stop_names,
}
# ---------------------------------------------------------------------------
# Init endpoint
# ---------------------------------------------------------------------------
@router.post("/init")
async def init_multi_agent(request: Request, payload: dict = Body(...)) -> StreamingResponse:
"""Streaming /init.
Emits text/event-stream frames as each seat/coordinator becomes ready, so
the frontend can visualize the parallel bring-up rather than blocking on
one bulk JSON. Frames:
data: {"event":"seat_ready","agent_name":"...","thread_id":"...","display_name":"...","description":"..."}
data: {"event":"coordinator_ready","main_agent_name":"...","thread_id":"..."}
data: {"event":"init_done","main_agent_name":"...","thread_ids":{...}}
data: {"event":"error","detail":"..."} # on failure
Validation errors (missing agent_names, bad id pattern) still raise
HTTPException synchronously so the connection fails fast with a real 4xx
instead of a 200 SSE that just carries an error frame.
"""
agent_names_in = payload.get("agent_names") or []
agent_names: list[str] = [str(name).lower() for name in agent_names_in]
main_agent_name: str = str(payload.get("main_agent_name") or "main-agent").lower()
model: str | None = payload.get("model")
if not agent_names:
raise HTTPException(status_code=400, detail="agent_names is required")
for name in agent_names:
_validate_agent_id(name, "agent_names entry")
_validate_agent_id(main_agent_name, "main_agent_name")
base = _loopback_base(request)
headers = _auth_headers(request)
def _frame(payload: dict[str, Any]) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
async def stream_generator() -> AsyncIterator[str]:
async with httpx.AsyncClient(timeout=None) as client:
try:
# Per-seat prepare task — same body as before, but we no longer
# await them with asyncio.gather. Instead, asyncio.as_completed
# surfaces them in REAL completion order so we can emit a
# seat_ready frame the moment each finishes.
async def _prepare_sub_agent(sub_id: str) -> tuple[str, str, str, str]:
info = await _get_agent(client, base, headers, sub_id)
display_name = str(info.get("name") or sub_id)
description = info.get("description", "") or ""
thread_id = await _create_thread(client, base, headers)
return sub_id.lower(), thread_id, display_name, description
tasks = [asyncio.create_task(_prepare_sub_agent(sid)) for sid in agent_names]
prepared: list[tuple[str, str, str, str]] = []
try:
for fut in asyncio.as_completed(tasks):
sid, thread_id, display_name, description = await fut
prepared.append((sid, thread_id, display_name, description))
yield _frame({
"event": "seat_ready",
"agent_name": sid,
"thread_id": thread_id,
"display_name": display_name,
"description": description,
})
except Exception:
# Cancel any still-pending sibling so we don't leak tasks
# past the AsyncClient close.
for t in tasks:
if not t.done():
t.cancel()
raise
agent_threads: dict[str, str] = {sid: thread_id for sid, thread_id, _, _ in prepared}
descriptions: list[str] = [
f"- agent_name: {sid}({display_name}):{description}"
for sid, _, display_name, description in prepared
]
soul = (
"#核心定位:\n"
"用途:根据用户的任务目标,调用其他智能体干活,只负责协调汇总,"
"当你觉得已有信息支持回答用户问题时候,停止调度,并输出最后的结果\n"
"#调度方法:\n"
"通过使用agent_orchestration技能调用其他智能体干活,使用技能时传入agent_name和需要做的活。"
"注意:agent_name 必须严格使用下方列出的英文 id(例如 roundtable-intelligence),"
"不要把括号里的中文显示名拼进 agent_name 里。\n"
"#可调度智能体:\n"
+ "\n".join(descriptions)
)
coord_thread = await _create_thread(client, base, headers)
agent_threads[main_agent_name] = coord_thread
ok, result = await _create_agent(
client,
base,
headers,
name=main_agent_name,
description="你是总控控制协调智能体,负责协调其他智能体干活",
model=model,
skills=["agent_orchestration"],
soul=soul,
agent_id=main_agent_name,
)
if not ok and isinstance(result, str) and "exists" not in result:
yield _frame({"event": "error", "detail": f"create coordinator agent failed: {result}"})
return
yield _frame({
"event": "coordinator_ready",
"main_agent_name": main_agent_name,
"thread_id": coord_thread,
})
yield _frame({
"event": "init_done",
"main_agent_name": main_agent_name,
"thread_ids": agent_threads,
})
except HTTPException as exc:
logger.warning("multi-agent init HTTPException: %s %s", exc.status_code, exc.detail)
yield _frame({"event": "error", "detail": f"[{exc.status_code}] {exc.detail}"})
except Exception as exc:
logger.exception("multi-agent init unexpected failure")
yield _frame({"event": "error", "detail": f"unexpected: {exc!r}"})
return StreamingResponse(
stream_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
# ---------------------------------------------------------------------------
# Stream endpoint
# ---------------------------------------------------------------------------
async def _stream_upstream(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
thread_id: str,
body: dict[str, Any],
) -> AsyncIterator[tuple[str | None, list[str] | None]]:
"""Yield (raw_line, None) for each SSE line, then ``(None, captured_data_lines)`` once."""
url = f"{base}/api/threads/{thread_id}/runs/stream"
captured: list[str] = []
t_send = time.perf_counter()
async with client.stream("POST", url, json=body, headers=headers) as resp:
t_headers = time.perf_counter()
if resp.status_code != 200:
body_text = (await resp.aread()).decode("utf-8", "replace")[:1000]
raise HTTPException(
status_code=502,
detail=f"upstream {url} returned HTTP {resp.status_code}: {body_text}",
)
agent_for_log = (body.get("context") or {}).get("agent_name") or thread_id
logger.info(
"[upstream-timing] %s headers_in=%.3fs",
agent_for_log,
t_headers - t_send,
)
first_line = True
async for line in resp.aiter_lines():
if not line:
continue
if first_line:
first_line = False
logger.info(
"[upstream-timing] %s first_line=%.3fs (LangGraph runtime startup + LLM first byte)",
agent_for_log,
time.perf_counter() - t_headers,
)
yield line, None
if line.startswith("data:"):
captured.append(line)
yield None, captured
def _parse_last_messages(captured: list[str]) -> list[dict[str, Any]]:
"""Return the full ``messages`` list from the most recent ``values`` snapshot.
The upstream sends the final ``values`` frame second-to-last (the very last
line is the ``end`` event with ``null``). We tolerate trailing non-JSON
frames (e.g. ``data: null``) by scanning backwards for the first frame that
successfully parses as a state object with a ``messages`` key.
"""
if not captured:
raise HTTPException(status_code=502, detail="upstream produced no data lines")
for raw_line in reversed(captured):
raw = raw_line[5:].strip() if raw_line.startswith("data:") else raw_line.strip()
if not raw or raw == "null":
continue
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict) and "messages" in parsed:
messages = parsed.get("messages") or []
if isinstance(messages, list):
return messages
raise HTTPException(status_code=502, detail="no parseable values frame in upstream stream")
def _find_trailing_ai_text(messages: list[dict[str, Any]], *, exclude_text: str = "") -> str:
"""Return the text content of the latest AIMessage with no tool_calls.
Used by the clarification branch: ClarificationMiddleware ends the run
after emitting the structured ToolMessage, but the model often produces a
richer human-readable follow-up AIMessage right before the interrupt.
Prefer that text over the brief preamble that accompanied the
ask_clarification tool_call. When the only trailing AIMessage matches the
preamble (``exclude_text``), return empty so the caller can fall back to
the question itself.
"""
for msg in reversed(messages):
if not isinstance(msg, dict):
continue
if (msg.get("type") or "").lower() not in ("ai", "aimessage", "aimessagechunk"):
continue
if msg.get("tool_calls"):
continue
text = _flatten_content(msg.get("content", "")).strip()
if not text:
continue
if exclude_text and text == exclude_text.strip():
continue
return text
return ""
def _find_dispatch_message(messages: list[dict[str, Any]]) -> dict[str, Any]:
"""Locate the latest AIMessage that carries dispatch tool calls.
The very last message can be a skill-stop ToolMessage injected by
``SkillStopMiddleware`` when it intercepts the dispatch, or a clarification
ToolMessage. Either way, the actionable tool_calls live on the AIMessage
*before* it, so we walk backwards until we hit one with ``tool_calls``.
Returns the latest AIMessage when none of them carry tool_calls so the
caller can still extract its text content.
"""
last_ai: dict[str, Any] = {}
for msg in reversed(messages):
if not isinstance(msg, dict):
continue
if (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
if not last_ai:
last_ai = msg
if msg.get("tool_calls"):
return msg
return last_ai
async def _leader_run(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
agent_threads: dict[str, str],
agent_name: str,
new_message: str,
skill_stop_names: list[str],
model_name: str,
) -> AsyncIterator[str | dict[str, Any]]:
# Fine-grained timing for the leader path. The user reported the
# coordinator "stalls before output" — we log each major await so we can
# see whether the wait is in pre-broadcast scheduling, upstream stream
# setup, first-token latency, or post-stream parsing.
t0 = time.perf_counter()
pre_broadcast_task = asyncio.create_task(
_broadcast(client, base, headers, "总控智能体收到用户消息:" + new_message, agent_threads, [agent_name])
)
thread_id = agent_threads[agent_name]
effective_stop_names = list({"agent_orchestration", *skill_stop_names})
body = _run_payload(
agent_name=agent_name,
model_name=model_name,
thread_id=thread_id,
new_message=new_message,
excluded_tools=["web_search"],
skill_stop_names=effective_stop_names,
)
t_payload_built = time.perf_counter()
logger.info(
"[leader-timing] %s payload_built=%.3fs (broadcast running in background)",
agent_name,
t_payload_built - t0,
)
captured: list[str] = []
first_line_ts: float | None = None
line_count = 0
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
if line is not None:
if first_line_ts is None:
first_line_ts = time.perf_counter()
logger.info(
"[leader-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
agent_name,
first_line_ts - t_payload_built,
first_line_ts - t0,
)
line_count += 1
yield line
elif done is not None:
captured = done
t_stream_done = time.perf_counter()
logger.info(
"[leader-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
agent_name,
t_stream_done - (first_line_ts or t_payload_built),
line_count,
t_stream_done - t0,
)
messages = _parse_last_messages(captured)
# Pre-broadcast must finish before we yield the final status frame on any
# branch — the seats that get dispatched next rely on having seen the
# original user message. The upstream LLM stream usually takes several
# seconds, so the broadcast has typically already completed and this
# await is effectively free.
t_before_broadcast_await = time.perf_counter()
try:
await pre_broadcast_task
except Exception as exc:
logger.warning("pre-broadcast failed: %s", exc)
t_after_broadcast_await = time.perf_counter()
logger.info(
"[leader-timing] %s pre_broadcast_await=%.3fs (>0 means broadcast was still in flight after LLM finished)",
agent_name,
t_after_broadcast_await - t_before_broadcast_await,
)
dispatch_msg = _find_dispatch_message(messages)
leader_text = _flatten_content(dispatch_msg.get("content", ""))
tool_calls = dispatch_msg.get("tool_calls") or []
# Clarification branch: the coordinator called ask_clarification. The
# ClarificationMiddleware intercepts with Command(goto=END) and stashes
# the structured question in the tool_call args; sometimes a richer
# follow-up AIMessage is also produced after the ToolMessage. Emit a
# dedicated frame so the frontend can render it as a question instead of
# treating an empty dispatch as "consensus reached".
clarification_call = next(
(tc for tc in tool_calls if (tc.get("name") or "").lower() == "ask_clarification"),
None,
)
if clarification_call is not None:
cargs = clarification_call.get("args") or {}
question = str(cargs.get("question") or "").strip()
follow_up = _find_trailing_ai_text(messages, exclude_text=leader_text)
yield {
"status": "clarification",
"content": follow_up or question or leader_text,
"question": question,
"clarification_type": str(cargs.get("clarification_type") or ""),
"clarification_context": str(cargs.get("context") or ""),
"options": cargs.get("options") or [],
"allow_custom": bool(cargs.get("allow_custom", True)),
"agent_name": agent_name,
}
return
if not tool_calls:
yield {"status": [], "content": leader_text, "agent_name": agent_name}
return
# Run all dispatch broadcasts concurrently — they target disjoint thread
# sets and we don't care about ordering, so serializing them just
# multiplied latency before the frontend could start a sub-agent run.
dispatched: list[list[str]] = []
dispatch_broadcasts: list[asyncio.Task[None]] = []
for tool_call in tool_calls:
try:
sub_name, task_text = _extract_dispatch(tool_call)
if not sub_name:
continue
dispatch_broadcasts.append(
asyncio.create_task(
_broadcast(
client,
base,
headers,
f"总控智能体给{sub_name}派活,内容为:" + task_text,
agent_threads,
[agent_name, sub_name],
)
)
)
dispatched.append([sub_name, task_text])
except Exception as exc:
logger.warning("dispatch failed: %s", exc)
if dispatch_broadcasts:
t_before_dispatch = time.perf_counter()
await asyncio.gather(*dispatch_broadcasts, return_exceptions=True)
logger.info(
"[leader-timing] %s dispatch_broadcasts_await=%.3fs count=%d (gates how soon the next sub agent can start)",
agent_name,
time.perf_counter() - t_before_dispatch,
len(dispatch_broadcasts),
)
yield {"status": dispatched, "content": leader_text, "agent_name": agent_name}
async def _special_run(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
agent_threads: dict[str, str],
agent_name: str,
new_message: str,
skill_stop_names: list[str],
model_name: str,
) -> AsyncIterator[str | dict[str, Any]]:
# Same instrumentation as _leader_run so all 6 seats are diagnosable.
t0 = time.perf_counter()
thread_id = agent_threads[agent_name]
body = _run_payload(
agent_name=agent_name,
model_name=model_name,
thread_id=thread_id,
new_message=new_message,
excluded_tools=["web_search", "ask_clarification", "present_files", "view_image", "agent_orchestration"],
skill_stop_names=skill_stop_names,
)
t_payload_built = time.perf_counter()
logger.info(
"[special-timing] %s payload_built=%.3fs",
agent_name,
t_payload_built - t0,
)
captured: list[str] = []
first_line_ts: float | None = None
line_count = 0
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
if line is not None:
if first_line_ts is None:
first_line_ts = time.perf_counter()
logger.info(
"[special-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
agent_name,
first_line_ts - t_payload_built,
first_line_ts - t0,
)
line_count += 1
yield line
elif done is not None:
captured = done
t_stream_done = time.perf_counter()
logger.info(
"[special-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
agent_name,
t_stream_done - (first_line_ts or t_payload_built),
line_count,
t_stream_done - t0,
)
messages = _parse_last_messages(captured)
# Sub-agent's final deliverable is the latest AIMessage's text content.
last_msg: dict[str, Any] = {}
for msg in reversed(messages):
if isinstance(msg, dict) and (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
last_msg = msg
break
content = _flatten_content(last_msg.get("content", ""))
# Kick the "子智能体完成" broadcast off in the background and yield the
# status frame first — the next leader cycle that depends on this
# broadcast having landed only starts after a fresh /run/stream request
# from the frontend, which is many seconds away (full leader LLM call).
# Awaiting after the yield ensures the task finishes before the AsyncClient
# closes; the SSE connection stays open a bit longer but the frontend has
# already consumed the result.
t_before_post = time.perf_counter()
post_broadcast_task = asyncio.create_task(
_broadcast(
client,
base,
headers,
f"子智能体{agent_name}完成{new_message}工作,交付内容为:" + content,
agent_threads,
[agent_name],
)
)
yield {
"status": f"子智能体{agent_name}完成{new_message}工作",
"content": content,
"agent_name": agent_name,
}
try:
await post_broadcast_task
except Exception as exc:
logger.warning("post-broadcast failed: %s", exc)
logger.info(
"[special-timing] %s post_broadcast_total=%.3fs (broadcast time after yield — keeps SSE open this long)",
agent_name,
time.perf_counter() - t_before_post,
)
@router.post("/run/stream")
async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) -> StreamingResponse:
agent_type: str = str(payload.get("agent_type") or "leader").lower()
agent_threads: dict[str, str] = payload.get("d_agent_thread_id") or {}
agent_name: str = str(payload.get("agent_name") or "").lower()
new_message: str = str(payload.get("new_message") or "")
skill_stop_names: list[str] = payload.get("skill_stop_names") or []
model_name: str = str(payload.get("model") or "deepseek-chat")
if not agent_name or agent_name not in agent_threads:
raise HTTPException(
status_code=400,
detail=f"agent_name '{agent_name}' is missing from d_agent_thread_id",
)
_validate_agent_id(agent_name, "agent_name")
for key in agent_threads:
_validate_agent_id(key, "d_agent_thread_id key")
base = _loopback_base(request)
headers = _auth_headers(request)
async def stream_generator() -> AsyncIterator[str]:
async with httpx.AsyncClient(timeout=None) as client:
try:
if agent_type == "leader":
gen = _leader_run(
client, base, headers, agent_threads, agent_name, new_message, skill_stop_names, model_name,
)
else:
gen = _special_run(
client, base, headers, agent_threads, agent_name, new_message, skill_stop_names, model_name,
)
async for chunk in gen:
if isinstance(chunk, dict):
yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"
else:
yield chunk if chunk.endswith("\n") else chunk + "\n"
except HTTPException as exc:
logger.warning("multi-agent stream HTTPException: %s %s", exc.status_code, exc.detail)
err_frame = {
"status": "error",
"content": f"[{exc.status_code}] {exc.detail}",
"agent_name": agent_name,
}
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
except Exception as exc:
logger.exception("multi-agent stream unexpected failure")
err_frame = {
"status": "error",
"content": f"unexpected: {exc!r}",
"agent_name": agent_name,
}
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
return StreamingResponse(
stream_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
@@ -0,0 +1,385 @@
"""Public (unauthenticated) API for scheduled tasks.
Mounted under the ``/api/public/`` prefix so the auth middleware lets these
routes through without a session cookie. Intended for embedding scheduled
task results into external dashboards / documentation / chat systems that
shouldn't need a DeerFlow login.
Surface area:
- ``GET /api/public/scheduled-tasks/`` — list every task (no prompt body)
- ``GET /api/public/scheduled-tasks/{task_id}`` — task metadata
- ``GET /api/public/scheduled-tasks/{task_id}/runs`` — runs for a task (latest first)
- ``GET /api/public/scheduled-tasks/runs/{run_id}`` — single run (result included)
- ``GET /api/public/scheduled-tasks/runs/{run_id}/files/{i}``— inline file content / preview
- ``PUT /api/public/scheduled-tasks/runs/{run_id}/files/{i}``— overwrite an editable result file
- ``PUT /api/public/scheduled-tasks/runs/{run_id}/body`` — overwrite the inline Markdown body
- ``GET /api/public/scheduled-tasks/runs/{run_id}/download`` — zip / md download of result
- ``DELETE /api/public/scheduled-tasks/runs/{run_id}`` — remove a single run (no auth)
The PUT endpoints intentionally skip ownership checks so the anonymous public
viewer at ``#/public/scheduled-tasks/<task_id>`` can fix typos in result MD
files / body content. Tighten this if the deployment requires read-only
sharing.
"""
from __future__ import annotations
import logging
import mimetypes
import re
import zipfile
from io import BytesIO
from pathlib import Path
from typing import Any
from urllib.parse import quote
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from deerflow.runtime.scheduler import get_scheduled_task_service
logger = logging.getLogger(__name__)
# Whitelisted under ``auth_middleware._PUBLIC_PATH_PREFIXES`` (``/api/public/``).
router = APIRouter(prefix="/api/public/scheduled-tasks", tags=["public-scheduled-tasks"])
_MAX_PUBLIC_EDIT_BYTES = 5 * 1024 * 1024 # 5 MB — mirrors the authenticated edit cap
class PublicRunFileUpdateRequest(BaseModel):
content: str = Field(..., description="New text content to write into the run file")
class PublicRunBodyUpdateRequest(BaseModel):
content: str = Field(..., description="New Markdown body for the run's result")
# ---------------------------------------------------------------------------
# Sanitisation helpers
# ---------------------------------------------------------------------------
# Fields stripped from every task before it leaves the public boundary.
# Keeps owner ids, raw cron, and the (often verbose) author prompt out of
# anonymous responses.
_TASK_PRIVATE_FIELDS: frozenset[str] = frozenset(
{
"user_id",
"owner_user_id",
"creator_user_id",
"creator_email",
"prompt",
"execution_context",
"_subscription",
"role",
"scheduler_thread_id",
}
)
# Fields stripped from every run before it leaves the public boundary.
# We keep the result body (the user explicitly wants the MD content surfaced)
# but redact any path that points at a server-side filesystem location.
_RUN_PRIVATE_FIELDS: frozenset[str] = frozenset(
{
"user_id",
"owner_user_id",
"scheduler_thread_id",
"agent_run_id",
"can_edit",
}
)
def _redact_task(task: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in task.items() if k not in _TASK_PRIVATE_FIELDS}
def _redact_run(run: dict[str, Any]) -> dict[str, Any]:
cleaned = {k: v for k, v in run.items() if k not in _RUN_PRIVATE_FIELDS}
result = cleaned.get("result")
if isinstance(result, dict):
files = result.get("files")
if isinstance(files, list):
cleaned_files = []
for idx, item in enumerate(files):
if not isinstance(item, dict):
continue
# Strip absolute filesystem path; clients use the file index
# to fetch content via the public file endpoint.
public_file = {k: v for k, v in item.items() if k != "path"}
public_file["index"] = idx
# Mark whether the underlying archive is still on disk. The
# public viewer filters out tabs whose source is gone so users
# never see the raw "Result file is no longer available" 404.
path_value = item.get("path")
public_file["available"] = (
isinstance(path_value, str)
and bool(path_value)
and Path(path_value).is_file()
)
cleaned_files.append(public_file)
result = {**result, "files": cleaned_files}
cleaned["result"] = result
return cleaned
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _safe_filename(value: str, default: str = "scheduled-task-result") -> str:
name = re.sub(r'[\\/:*?"<>|\r\n]+', "-", value).strip(" .")
return name or default
def _result_markdown(result: dict[str, Any]) -> str:
title = str(result.get("title") or "Scheduled Task Result")
content = str(result.get("content") or "")
return f"# {title}\n\n{content}".strip() + "\n"
def _utf8_sig_bytes(content: str) -> bytes:
return content.encode("utf-8-sig")
def _content_disposition(filename: str) -> str:
quoted = quote(filename)
return f"attachment; filename*=UTF-8''{quoted}"
def _run_file_path(run: dict[str, Any], file_index: int) -> tuple[dict[str, Any], Path]:
result = run.get("result") or {}
files = result.get("files") if isinstance(result, dict) and isinstance(result.get("files"), list) else []
if file_index < 0 or file_index >= len(files):
raise HTTPException(status_code=404, detail="Result file not found")
item = files[file_index]
if not isinstance(item, dict) or not isinstance(item.get("path"), str):
raise HTTPException(status_code=404, detail="Result file not found")
path = Path(item["path"])
if not path.exists() or not path.is_file():
raise HTTPException(status_code=404, detail="Result file is no longer available")
return item, path
def _is_text_previewable(path: Path, mime_type: str | None) -> bool:
if mime_type and (mime_type.startswith("text/") or mime_type in {"application/json", "application/xml"}):
return True
return path.suffix.lower() in {".md", ".markdown", ".txt", ".json", ".csv", ".xml", ".yaml", ".yml", ".log"}
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
@router.get("")
async def public_list_tasks(
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on task name"),
limit: int = Query(default=200, ge=1, le=500),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
"""Return every scheduled task across all users.
Sensitive fields (user_id, raw prompt, scheduler thread id, …) are stripped
before serialisation. The response shape is intentionally stable so external
integrations can pin to it without surprises.
"""
service = get_scheduled_task_service()
tasks = await service.admin_list_tasks()
if search:
needle = search.strip().lower()
tasks = [t for t in tasks if needle in str(t.get("name", "")).lower()]
total = len(tasks)
sliced = tasks[offset : offset + limit]
return {
"total": total,
"limit": limit,
"offset": offset,
"tasks": [_redact_task(task) for task in sliced],
}
@router.get("/{task_id}")
async def public_get_task(task_id: str) -> dict[str, Any]:
"""Return metadata for a single task (without prompt body)."""
service = get_scheduled_task_service()
task = await service.store.get_task_any(task_id)
if task is None:
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
return _redact_task(task)
@router.get("/{task_id}/runs")
async def public_list_task_runs(
task_id: str,
limit: int = Query(default=50, ge=1, le=200),
) -> dict[str, Any]:
"""Return runs for a task, latest first."""
service = get_scheduled_task_service()
task = await service.store.get_task_any(task_id)
if task is None:
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
runs = await service.admin_list_runs(task_id, limit=limit)
return {
"task": _redact_task(task),
"total": len(runs),
"runs": [_redact_run(run) for run in runs],
}
@router.get("/runs/{run_id}")
async def public_get_run(run_id: str) -> dict[str, Any]:
"""Return a single run with its full result body (Markdown included)."""
service = get_scheduled_task_service()
run = await service.admin_get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return _redact_run(run)
@router.get("/runs/{run_id}/files/{file_index}")
async def public_get_run_file(run_id: str, file_index: int) -> dict[str, Any]:
"""Return a single result-file entry with its inline text content where
previewable. Binary files come back with ``previewable: false`` and the
consumer should fetch the download endpoint instead."""
service = get_scheduled_task_service()
run = await service.admin_get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
item, path = _run_file_path(run, file_index)
mime_type = item.get("mime_type") or mimetypes.guess_type(path.name)[0]
previewable = _is_text_previewable(path, mime_type)
content: str | None = None
if previewable:
try:
content = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
content = path.read_text(encoding="utf-8", errors="replace")
return {
"index": file_index,
"name": item.get("name") or path.name,
"mime_type": mime_type,
"kind": item.get("kind"),
"size": path.stat().st_size,
"previewable": previewable,
"content": content,
}
@router.get("/runs/{run_id}/files/{file_index}/download")
async def public_download_run_file(run_id: str, file_index: int):
"""Stream a single result file as a download."""
service = get_scheduled_task_service()
run = await service.admin_get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
item, path = _run_file_path(run, file_index)
filename = _safe_filename(str(item.get("name") or path.name), path.name)
media = item.get("mime_type") or mimetypes.guess_type(path.name)[0] or "application/octet-stream"
return FileResponse(path, filename=filename, media_type=media)
@router.get("/runs/{run_id}/download")
async def public_download_run(run_id: str):
"""Download the full result: a zipped bundle of result.md + every file,
or a single ``.md`` when no auxiliary files were produced."""
service = get_scheduled_task_service()
run = await service.admin_get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
result = run.get("result") or {}
if not isinstance(result, dict):
result = {}
files = result.get("files") if isinstance(result.get("files"), list) else []
accessible_files: list[tuple[str, Path]] = []
for item in files:
if not isinstance(item, dict):
continue
path_value = item.get("path")
if not isinstance(path_value, str) or not path_value:
continue
path = Path(path_value)
if path.exists() and path.is_file():
accessible_files.append((str(item.get("name") or path.name), path))
base_name = _safe_filename(str(result.get("title") or "scheduled-task-result"))
if not accessible_files:
return Response(
content=_utf8_sig_bytes(_result_markdown(result)),
media_type="text/markdown; charset=utf-8",
headers={"Content-Disposition": _content_disposition(f"{base_name}.md")},
)
archive = BytesIO()
with zipfile.ZipFile(archive, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
zf.writestr("result.md", _utf8_sig_bytes(_result_markdown(result)))
for name, path in accessible_files:
arcname = _safe_filename(name, path.name)
zf.write(path, arcname=arcname)
archive.seek(0)
return StreamingResponse(
archive,
media_type="application/zip",
headers={"Content-Disposition": _content_disposition(f"{base_name}.zip")},
)
@router.put("/runs/{run_id}/files/{file_index}")
async def public_update_run_file(
run_id: str,
file_index: int,
body: PublicRunFileUpdateRequest,
) -> dict[str, Any]:
"""Overwrite an MD/text result file on disk and refresh its size in the run row.
Anonymous — no ownership check. See module docstring for the rationale.
"""
if len(body.content.encode("utf-8")) > _MAX_PUBLIC_EDIT_BYTES:
raise HTTPException(status_code=413, detail="Edited content exceeds the 5 MB limit")
service = get_scheduled_task_service()
try:
return await service.admin_update_run_file(run_id, file_index, body.content)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc))
except PermissionError as exc:
# Surfaces archive-path escape attempts. Treat as 400 since the caller is
# already trusted to be reaching a real run row.
raise HTTPException(status_code=400, detail=str(exc))
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
@router.delete("/runs/{run_id}", status_code=204)
async def public_delete_run(run_id: str) -> Response:
"""Remove a single run regardless of owner.
Anonymous — no ownership check, mirroring the existing public PUT endpoints.
Tighten this if the deployment requires read-only public sharing.
"""
service = get_scheduled_task_service()
deleted = await service.admin_delete_run(run_id)
if not deleted:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return Response(status_code=204)
@router.put("/runs/{run_id}/body")
async def public_update_run_body(run_id: str, body: PublicRunBodyUpdateRequest) -> dict[str, Any]:
"""Overwrite the inline Markdown body (``result.content``) of a run.
Anonymous — no ownership check. Used by the public viewer's "正文" tab.
"""
if len(body.content.encode("utf-8")) > _MAX_PUBLIC_EDIT_BYTES:
raise HTTPException(status_code=413, detail="Edited content exceeds the 5 MB limit")
service = get_scheduled_task_service()
try:
return await service.admin_update_run_body(run_id, body.content)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc))
@@ -14,7 +14,16 @@ from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import FileResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_local_provider, get_optional_user_from_request, get_run_event_store, get_run_store, get_tag_store
from app.gateway.deps import (
get_checkpointer,
get_current_user,
get_local_provider,
get_optional_user_from_request,
get_run_event_store,
get_run_store,
get_tag_store,
)
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.scheduler import DuplicateTaskNameError, get_scheduled_task_service
router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
@@ -22,6 +31,7 @@ router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
class ScheduledTaskCreateRequest(BaseModel):
name: str = Field(default="Scheduled task")
description: str | None = None
prompt: str
schedule: str
timezone: str | None = None
@@ -31,6 +41,7 @@ class ScheduledTaskCreateRequest(BaseModel):
class ScheduledTaskUpdateRequest(BaseModel):
name: str | None = None
description: str | None = None
prompt: str | None = None
schedule: str | None = None
timezone: str | None = None
@@ -235,31 +246,129 @@ async def get_scheduled_task_run(run_id: str, request: Request) -> dict:
return run
def _event_type_for_checkpoint_message(message: dict) -> str:
"""Map a serialized LangChain message to the same ``event_type`` strings
the run-event store uses, so the frontend's existing role labels keep
working for checkpoint-derived synthetic rows."""
raw_type = message.get("type") or message.get("role") or ""
if raw_type in {"human", "user"}:
return "llm.human.input"
if raw_type in {"ai", "assistant"}:
return "llm.ai.response"
if raw_type == "tool":
return "llm.tool.result"
if raw_type == "system":
return "llm.system"
return f"llm.{raw_type or 'message'}"
async def _checkpoint_messages_for_run(
request: Request, agent_run: dict, *, limit: int
) -> list[dict]:
"""Return a synthetic list of message rows pulled from the LangGraph
checkpoint for an in-flight run.
The event store is updated by the streaming pipeline and may stay empty
until the first message tuple is flushed — meanwhile the checkpoint
already carries the current state (input + accumulating AI deltas). We
project that into the same row shape the frontend uses for the event
store, so the run-detail page can show progress immediately."""
try:
checkpointer = get_checkpointer(request)
except Exception:
return []
thread_id = agent_run.get("thread_id")
if not thread_id:
return []
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
tup = await checkpointer.aget_tuple(config)
except Exception:
return []
if tup is None:
return []
checkpoint = getattr(tup, "checkpoint", {}) or {}
channel_values = checkpoint.get("channel_values") or {}
try:
values = serialize_channel_values(channel_values)
except Exception:
return []
messages = values.get("messages") or []
rows: list[dict] = []
for index, message in enumerate(messages[-limit:]):
if not isinstance(message, dict):
continue
rows.append(
{
"seq": index + 1,
"event_type": _event_type_for_checkpoint_message(message),
"category": "message",
"content": message,
"metadata": {"source": "checkpoint"},
"created_at": None,
}
)
return rows
@router.get("/runs/{run_id}/messages")
async def get_scheduled_task_run_messages(run_id: str, request: Request, limit: int = 500) -> dict:
"""Agent messages produced by a scheduled run's underlying agent run.
Lets the run-detail page show what the agent is doing — useful when a run
is stuck in ``running``. Authorization reuses :func:`_load_run` (owner /
subscriber / admin); the event-store query then passes ``user_id=None``
because access has already been checked.
Resolution order (most authoritative first):
1. Event-store rows for the underlying agent run — these are the per-tuple
streaming events the worker writes as the agent runs.
2. LangGraph checkpoint snapshot — the canonical conversation state. Used
as a fallback when the event store is still empty (early in a run) or
when only partial events have been flushed; we project the snapshot
into the same row shape the frontend already renders so the user can
see the prompt + current AI output while the task is mid-flight.
3. Synthetic human-input row built from ``task.prompt`` — shown even
before the agent run is registered, so a `pending` task at minimum
displays what was queued.
"""
service = get_scheduled_task_service()
user_id = await _user_id(request)
run = await _load_run(service, request, run_id, user_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
capped = max(1, min(limit, 1000))
# Always available: task prompt as the seeded human input. Lets the
# ``execution process`` tab show *something* even before the agent run is
# registered, which is the user-visible bug we're fixing.
fallback_rows: list[dict] = []
task_id = run.get("task_id")
if task_id:
task = await service.store.get_task_any(task_id)
prompt = (task or {}).get("prompt")
if prompt:
fallback_rows.append(
{
"seq": 0,
"event_type": "llm.human.input",
"category": "message",
# Frontend treats human-input content as a plain string.
"content": str(prompt),
"metadata": {"source": "task_prompt"},
"created_at": run.get("created_at") or run.get("scheduled_for"),
}
)
agent_run_id = run.get("agent_run_id")
if not agent_run_id:
return {"messages": [], "has_more": False}
return {"messages": fallback_rows, "has_more": False}
# Each scheduled run executes on its own throwaway runtime thread — NOT the
# run row's ``scheduler_thread_id`` — so resolve the real thread from the
# run store before querying its message events.
run_store = get_run_store(request)
agent_run = await run_store.get(agent_run_id, user_id=None)
if agent_run is None:
return {"messages": [], "has_more": False}
capped = max(1, min(limit, 1000))
return {"messages": fallback_rows, "has_more": False}
event_store = get_run_event_store(request)
rows = await event_store.list_messages_by_run(
agent_run["thread_id"],
@@ -267,6 +376,16 @@ async def get_scheduled_task_run_messages(run_id: str, request: Request, limit:
limit=capped + 1,
user_id=None,
)
# Event store has no rows yet (run just started, or events still buffering).
# Fall back to the live checkpoint so the user immediately sees the prompt
# + whatever AI output has been streamed so far.
if not rows:
checkpoint_rows = await _checkpoint_messages_for_run(request, agent_run, limit=capped)
if checkpoint_rows:
return {"messages": checkpoint_rows[:capped], "has_more": False}
return {"messages": fallback_rows, "has_more": False}
has_more = len(rows) > capped
return {"messages": rows[:capped] if has_more else rows, "has_more": has_more}
@@ -430,6 +549,7 @@ async def create_scheduled_task(body: ScheduledTaskCreateRequest, request: Reque
task = await service.create_task(
user_id=user_id,
name=body.name,
description=body.description,
prompt=body.prompt,
schedule=body.schedule,
timezone=body.timezone,
@@ -161,6 +161,13 @@ class SkillUploadValidateResponse(BaseModel):
warnings: list[str] = Field(default_factory=list, description="Non-fatal warnings to surface to the user")
class SkillReconcileResponse(BaseModel):
"""Response model for reconciling custom skills found on disk."""
adopted: list[str] = Field(default_factory=list, description="Skills found on disk that had no DB row; now owned by the caller")
already_registered: list[str] = Field(default_factory=list, description="Skills found on disk that already had a DB row (unchanged)")
class CustomSkillContentResponse(SkillResponse):
content: str = Field(..., description="Raw SKILL.md content")
@@ -483,6 +490,54 @@ async def install_skill(
raise HTTPException(status_code=500, detail=f"Failed to install skill: {str(e)}")
@router.post(
"/skills/reconcile-custom",
response_model=SkillReconcileResponse,
summary="Adopt Orphan Custom Skills",
description=(
"Scan the on-disk ``skills/custom/`` directory and register any skill that has a "
"valid SKILL.md but no DB ownership row. Adopted skills are recorded as owned by "
"the calling user with ``published=false``. Skips skills that already have a row "
"(regardless of owner). Useful after manually unzipping skills onto the filesystem."
),
)
async def reconcile_custom_skills(
request: Request,
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
) -> SkillReconcileResponse:
try:
user_id = _current_user_id(request)
storage = get_or_new_skill_storage(app_config=config)
custom_skills = [s for s in storage.load_skills(enabled_only=False) if s.category == SkillCategory.CUSTOM]
adopted: list[str] = []
already: list[str] = []
for skill in custom_skills:
existing = await skill_store.get_any(skill.name)
if existing is not None:
already.append(skill.name)
continue
await skill_store.ensure_legacy(
{"name": skill.name, "owner_user_id": user_id, "published": False}
)
try:
from deerflow.skills.usage import mark_uploaded
mark_uploaded(skill.name)
except Exception:
pass
adopted.append(skill.name)
if adopted:
await refresh_skills_system_prompt_cache_async()
return SkillReconcileResponse(adopted=adopted, already_registered=already)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to reconcile custom skills: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to reconcile custom skills: {e}")
@router.get("/skills/custom", response_model=SkillsListResponse, summary="List Custom Skills")
async def list_custom_skills(
request: Request,
@@ -54,6 +54,9 @@ class RunCreateRequest(BaseModel):
after_seconds: float | None = Field(default=None, description="Delayed execution")
if_not_exists: Literal["reject", "create"] = Field(default="create", description="Thread creation policy")
feedback_keys: list[str] | None = Field(default=None, description="LangSmith feedback keys")
excluded_tools: list[str] | None = Field(default=None, description="Tool names to exclude from the main agent (e.g. ['task', 'ask_clarification'])")
skill_stop_names: list[str] | None = Field(default=None, description="Skill names to stop execution when invoked (e.g. ['newsletter-generation'])")
skill_stop_message: str | None = Field(default=None, description="Custom message to display when skill stop is triggered")
class RunResponse(BaseModel):
@@ -100,7 +100,7 @@ _DEFAULT_ASSISTANT_ID = "lead_agent"
# Built-in LangGraph graph IDs registered in langgraph.json.
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent", "ai_writing"})
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
@@ -162,6 +162,9 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"max_concurrent_subagents",
"agent_name",
"is_bootstrap",
"excluded_tools",
"skill_stop_names",
"skill_stop_message",
}
)
@@ -186,15 +189,20 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
def resolve_agent_factory(assistant_id: str | None):
"""Resolve the agent factory callable from config.
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
to their own factory. Custom agents share the ``lead_agent`` factory with
``agent_id`` injected into configurable/context by :func:`build_run_config`.
Built-in graphs (``lead_agent``, ``canvas_agent``, ``ai_writing``) are dispatched
directly to their own factory. Custom agents share the ``lead_agent`` factory
with ``agent_id`` injected into configurable/context by :func:`build_run_config`.
"""
if assistant_id == "canvas_agent":
from deerflow.agents.canvas_agent.agent import make_canvas_agent
return make_canvas_agent
if assistant_id == "ai_writing":
from deerflow.agents.ai_writing.graph import make_ai_writing_graph
return make_ai_writing_graph
from deerflow.agents.lead_agent.agent import make_lead_agent
return make_lead_agent
@@ -400,6 +408,11 @@ async def start_run(
)
else:
await run_ctx.thread_store.update_status(thread_id, "running")
# SDK 的 ``client.threads.create()`` 不传 assistant_id —— 先建出来空的 thread,
# 真正提交 run 时才知道用哪个 assistant。这里回填一次,避免历史列表 / 按
# assistant 筛选时拿不到关联。
if not existing.get("assistant_id") and body.assistant_id:
await run_ctx.thread_store.update_assistant_id(thread_id, body.assistant_id)
except Exception:
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
@@ -424,7 +437,15 @@ async def start_run(
# The ``context`` field is a custom extension for the langgraph-compat layer
# that carries agent configuration (model_name, thinking_enabled, etc.).
# Only agent-relevant keys are forwarded; unknown keys (e.g. thread_id) are ignored.
merge_run_context_overrides(config, getattr(body, "context", None))
body_context = dict(getattr(body, "context", None) or {})
# Promote whitelisted top-level body fields into the context map so
# callers can pass them either nested under ``context`` or as flat
# top-level fields (``excluded_tools``, ``skill_stop_names``, …).
for top_level_key in ("excluded_tools", "skill_stop_names", "skill_stop_message"):
value = getattr(body, top_level_key, None)
if value is not None and top_level_key not in body_context:
body_context[top_level_key] = value
merge_run_context_overrides(config, body_context or None)
stream_modes = normalize_stream_modes(body.stream_mode)
@@ -441,6 +462,9 @@ async def start_run(
stream_subgraphs=body.stream_subgraphs,
interrupt_before=body.interrupt_before,
interrupt_after=body.interrupt_after,
# 透传 SDK ``Command``:resume / update / goto。Pregel 会据此从最近
# checkpoint 处恢复,而不是把 graph_input 当新一轮输入跑。
command=getattr(body, "command", None),
)
)
record.task = task
+58 -7
View File
@@ -41,6 +41,26 @@ models:
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: MiniMax-M2.7
display_name: MiniMax-M2.7
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.7
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com/v1
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: minimax-m2.5
display_name: MiniMax M2.5
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.5
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com/v1
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: false
title:
enabled: true
@@ -105,17 +125,40 @@ sandbox:
skills:
path: D:\project\bj\deerflow-server\skills
security_scan:
enabled: false
agents_api:
enabled: true
database:
backend: mysql
mysql_url: $MYSQL_DATABASE_URL
pool_size: 10
echo_sql: false
checkpointer:
type: sqlite
connection_string: D:\cursor-work\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
# 三层覆盖优先级(高->低):
# users/{user_id}/memory_config.json > .deer-flow/memory_user_overrides.json > 此处
memory:
enabled: true
version: v1 # 记忆版本: 'v1'(原版 DeerFlow, memory.json + LLM 自动提取)
# 'v2'(USER.md/MEMORY.md + 可选 Hindsight)
provider: hindsight # 'builtin' | 'hindsight'(外部 provider 名)
# builtin 始终启用,这里只决定 external 选哪个
# (version=v1 时此字段忽略)
# V1 专属配置 —— version=v1 时生效
v1:
max_facts: 100 # 最多保留的事实条数,超出时驱逐最低置信度的事实
fact_confidence_threshold: 0.7 # 事实置信度阈值,低于此值不保存
debounce_seconds: 30 # 对话结束后延迟多少秒才触发 LLM 提取
model_name: "" # 用于事实提取的 LLM 模型名;空则继承全局默认模型
token_budget: 2000 # 注入系统提示词的 token 预算
builtin:
enabled: true
memory_char_limit: 2200
@@ -124,15 +167,15 @@ memory:
hindsight: # provider=hindsight 时生效,需另行部署 Hindsight 服务
mode: local_external # 'cloud' | 'local_embedded' | 'local_external'
api_url: http://localhost:8765
api_url: http://47.88.25.99:18888
api_key: "" # 部署 Hindsight 后填,或改成 $HINDSIGHT_API_KEY 并设同名环境变量
# 注意:写成 $VAR 但环境变量未设置会导致配置加载失败
bank_id_template: "deerflow-user-{user_id}"
work_bank_id_template: "deerflow-work-{user_id}-{agent_id}"
memory_mode: context # 'context' | 'tools' | 'hybrid'
bank_mission: ""
bank_retain_mission: ""
recall_budget: mid # 'low' | 'mid' | 'high'
memory_mode: context # 'context' | 'tools' | 'hybrid'
bank_mission: "你是一个中文记忆助手,负责从对话中提炼和存储重要信息。所有记忆内容必须使用中文记录。"
bank_retain_mission: "请从以下对话中提炼重要信息,包括用户的偏好、需求、背景知识和关键事实。所有提炼的内容必须用中文表达。"
recall_budget: mid # 'low' | 'mid' | 'high'
recall_max_tokens: 4096
recall_max_input_chars: 800
auto_recall: true
@@ -158,10 +201,18 @@ memory:
max_tokens: 2000
include_builtin: true
include_hindsight: true
context_tag: "memory-context" # 包裹 Hindsight recall 的标签名
context_tag: "memory-context" # 包裹 Hindsight recall 的标签名
security:
scan_content: true # 注入/外渗模式扫描
scan_content: true # 注入/外渗模式扫描
block_invisible_unicode: true
# streaming_scrubber:预留字段,代码尚未实现 —— 待版本支持后取消注释。
# streaming_scrubber: true # SSE 流前清洗 <memory-context>
# AI 写作子系统 —— 素材收集专家专用配置。
# 注意:这三项必须嵌套在 ai_writing: 段下;写到顶层会被 _ai_writing_config()
# (app/gateway/routers/ai_writing.py:_ai_writing_config) 直接忽略。
ai_writing:
keyword_model: deepseek-chat # 检索词生成专用模型,留空回退写作主模型
rank_model: deepseek-chat # 素材排序专用模型,留空回退写作主模型
max_materials: 20 # 合并去重后的素材总量上限,范围 [1, 100],默认 20
@@ -0,0 +1,845 @@
# AI 写作 — 后端开发文档
> **本文档目的**:帮助后端开发者快速理解 AI 写作子系统的代码结构、Graph 拓扑、数据流、持久化、容错策略和扩展点。
>
> **配套文档**:
> - 前端:`frontend-web/docs/ai-writing-frontend-dev.md`
> - 故障排查:`frontend-web/docs/ai-writing-说明.md`
> - 前端 API 字段索引:`frontend-web/docs/ai-writing-assistant-backend-api.md`
---
## 一、产品定位与技术架构
### 产品定位
**杂志社协作写作子系统**:由 4 个 AI Agent(**素材收集专家 / 作家 / 编辑 / 系统**)协作完成一篇文章,通过 4 个用户干预点参与决策。
### 技术栈
- **编排框架**:LangGraph(独立 StateGraph,不复用主 lead agent 的 middleware 链)
- **节点驱动**:异步 LLM 调用(`langchain_openai` 等),支持流式 token 推送到 SSE
- **状态持久化**:LangGraph checkpointer(SQLite 文件,跨重启可恢复)
- **业务数据**:MySQL `ai_writing_sessions` 表
- **协议**:SSE(Server-Sent Events)推进度 + REST 控制流
### 与主 Agent 的关系
**完全独立**。AI 写作 graph 不通过 `make_lead_agent` 也不走主 agent 的 18 层 middleware。它有自己的 StateGraph、自己的 checkpointer、自己的路由前缀(`/api/ai-writing/*`),跟主 agent 解耦。
---
## 二、目录结构
```
offline-backend-20260512/backend/
├── app/gateway/
│ ├── routers/
│ │ └── ai_writing.py # FastAPI 路由 + 模块级状态字典 + SSE 生成器(1000+ 行,核心)
│ ├── ai_writing_cleanup.py # 定期清理 + cron 调度(独立模块)
│ └── app.py # lifespan 启动/停止清理任务、检测多 worker、关闭 checkpointer
│
└── packages/harness/deerflow/
├── agents/ai_writing/
│ ├── __init__.py # 暴露 AI_WRITING_GRAPH + 工厂函数
│ ├── graph.py # ⭐ StateGraph 定义 + 4 个 pause 节点 + checkpointer 工厂(538 行)
│ ├── state.py # AIWritingState TypedDict(121 行)
│ │
│ ├── nodes/ # ── Graph 节点 ──
│ │ ├── _utils.py # LLM 工具:parse_json, ThinkTagParser, llm_json
│ │ ├── intent_parser.py # 节点 1:意图解析
│ │ ├── researcher.py # 节点 2:素材检索(支持流式 progress)
│ │ ├── writer_outline.py # 节点 3:大纲规划
│ │ ├── writer_draft.py # 节点 4:草稿生成(按章节流式)
│ │ └── editor.py # 节点 5:编辑审核(3 维度并行流式)
│ │
│ ├── prompts/ # ── Prompt 模板 ──
│ │ ├── __init__.py
│ │ ├── researcher_prompts.py
│ │ ├── writer_prompts.py
│ │ └── editor_prompts.py
│ │
│ └── search/ # ── 素材检索 provider ──
│ ├── base.py # SearchResult / SearchProvider 接口
│ ├── duckduckgo_search.py
│ ├── internet_search.py # 互联网检索(默认)
│ ├── intranet_stub.py # 内网检索占位(按部署接入)
│ └── demo_search.py # 离线 demo 数据
│
└── persistence/ai_writing_sessions/
├── __init__.py # AIWritingSessionRepository 工厂
├── model.py # ORM:AIWritingSessionRow
└── sql.py # CRUD 实现(含清理用的 list_older_than / delete_by_ids)
```
---
## 三、Graph 拓扑(最重要)
### StateGraph 节点 + 边
```
START
│
↓
┌─────────────────┐
│ intent_parser │ 解析用户意图(确定 article_type / target_audience)
└────────┬────────┘
↓
┌─────────────────┐
│ researcher │ ← (re_search 回到这里)
└────────┬────────┘
↓ (after_researcher: ⼟材 ready)
┌──────────────────────┐
│ pause_material │ ⚙️ PAUSE-1:素材确认
└─┬──────────┬─────────┘
│ confirm │ re_search
↓ └→ researcher
┌─────────────────┐
│ writer_outline │ ← (re_outline 回到这里)
└────────┬────────┘
↓
┌──────────────────────┐
│ pause_outline │ ⚙️ PAUSE-2:大纲确认
└─┬──────────┬─────────┘
│ confirm │ re_outline
↓ └→ writer_outline
┌─────────────────┐
│ writer_draft │ ← (素材补充后回到这里)
└─┬───────────────┘
↓ (after_writer_draft)
┌──────┴────────────────┐
↓ ↓
┌──────────────────┐ ┌────────────────────┐
│ pause_section_ │ │ pause_draft │ ⚙️ PAUSE-3:草稿确认
│ help(素材不足)│ │ │
└─┬─────────────┬──┘ └─┬────────┬───┬─────┘
│ supplement │ del/ │ │ │ to_editor
│↓ │ loose │ │ user_revise
research writer_ │ final ↓
draft │ ize writer_draft
│↓
done
│
(to_editor)│
↓
┌─────────────────┐
│ editor │ 3 维度并行流式审核
└────────┬────────┘
↓ (after_review)
┌──────────────────────┐
│ pause_review │ ⚙️ PAUSE-4:审核确认
└─┬──────────┬─────────┘
│ confirm │ revise
│↓ └→ writer_draft
done
```
### 节点定义位置一览
| 节点名 | 实现文件 | 类型 | 流式? |
|---|---|---|---|
| `intent_parser` | `nodes/intent_parser.py` | async function | 否 |
| `researcher` | `nodes/researcher.py` | async function | 是(research_progress) |
| `pause_material` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
| `writer_outline` | `nodes/writer_outline.py` | async function | 是(outline_chunk) |
| `pause_outline` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
| `writer_draft` | `nodes/writer_draft.py` | async function | 是(draft_chunk,按章节) |
| `pause_section_help` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
| `pause_draft` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
| `editor` | `nodes/editor.py` | async function | 是(review_chunk,3 路并行) |
| `pause_review` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
### 条件路由函数(graph.py)
| 函数 | 在哪个节点之后 | 路由依据 |
|---|---|---|
| `after_researcher` | researcher | 是否找到素材 |
| `after_material_pause` | pause_material | `state["status"]` 是 researching 还是 writing_outline |
| `after_outline_pause` | pause_outline | 同上,writing_outline 还是 writing_draft |
| `after_writer_draft` | writer_draft | 是否有 blocked_sections |
| `after_section_help_pause` | pause_section_help | status |
| `after_draft_pause` | pause_draft | done / revising / reviewing |
| `after_review` | editor | review_result.verdict 是 pass 还是 reject + revision_count 上限 |
| `after_review_pause` | pause_review | done / writing_draft |
---
## 四、State 定义(state.py)
```python
class AIWritingState(TypedDict):
# —— 写作配置(启动时填充)——
user_intent: str
article_type: str
target_audience: str
word_count_target: int
keyword_count: int # 1-10,默认 6
writing_mode: str # "strict" | "loose"
# —— Agent 产出 ——
material_package: Optional[MaterialPackage] # 素材收集专家
current_outline: Optional[Outline] # 作家
drafts: Annotated[List[DraftArticle], operator.add] # 累计所有版本
current_draft: Optional[DraftArticle] # 最新草稿
review_result: Optional[ReviewResult] # 编辑
# —— 用户干预 ——
last_user_intervention: Optional[UserInterventionPayload]
blocked_sections: Optional[List[BlockedSection]] # 素材不足的章节
# —— 流程控制 ——
revision_count: int
max_revisions: int # 默认 3
status: WritingStatus
# —— SSE 推送队列 ——
progress_events: Annotated[List[dict], operator.add] # 累加,不覆盖
```
### State 字段的特殊语义
- **`Annotated[..., operator.add]`** — LangGraph 的累加 reducer,节点 return 时 `progress_events: [evt]` 会**追加**到现有列表,而不是替换
- **TypedDict** — 运行时是普通 dict,类型注解仅供 IDE 提示
- **节点输出** — 节点 return 的 dict 只需要包含**要更新的字段**,未指定的字段保持不变
---
## 五、路由层(app/gateway/routers/ai_writing.py)
### 14 个接口一览
| HTTP | 路径 | 用途 | 鉴权 |
|---|---|---|---|
| POST | `/api/ai-writing/start` | 启动写作,返回 session_id | 用户 |
| GET | `/api/ai-writing/{id}/stream` | SSE 进度流 | 用户 |
| POST | `/api/ai-writing/{id}/resume` | 提交用户干预 | 用户 |
| GET | `/api/ai-writing/{id}/state` | 查 graph 状态快照(调试用) | 用户 |
| GET | `/api/ai-writing/sessions` | 列出当前用户的历史会话 | 用户 |
| GET | `/api/ai-writing/sessions/{id}` | 获取单个历史会话 | 用户 |
| PATCH | `/api/ai-writing/sessions/{id}` | 重命名会话 | 用户 |
| DELETE | `/api/ai-writing/sessions/{id}` | 删除会话 | 用户 |
| PUT | `/api/ai-writing/sessions/{id}/transcript` | 保存对话时间线(**fire-and-forget**) | 用户 |
| GET | `/api/ai-writing/article-types` | 列出文章类型 | 用户 |
| POST | `/api/ai-writing/article-types` | 新增文章类型 | 用户(建议改成 admin) |
| PATCH | `/api/ai-writing/article-types/{id}` | 更新文章类型 | 用户 |
| DELETE | `/api/ai-writing/article-types/{id}` | 删除文章类型 | 用户 |
| POST | `/api/ai-writing/admin/cleanup` | **手动触发清理** | admin only |
### 模块级状态字典(关键设计 — 也是 worker 限制的根源)
```python
# 每个 session_id 对应一份内存状态。多 worker 部署时每个 worker 一份,互不共享。
_event_queues: dict[str, asyncio.Queue] # SSE 事件队列
_session_pending_pause: dict[str, tuple] # 挂起的 await_user 事件(供重连重投)
_session_model_names: dict[str, str] # 用户选的模型
_session_interventions: dict[str, list] # 累计干预历史
_session_repos: dict[str, repo] # DB 仓库引用
_session_user_ids: dict[str, str] # 用户 id
_session_article_types: dict[str, list] # 动态文章类型列表
_session_error_flags: dict[str, bool]
_inflight_graph_tasks: set[asyncio.Task] # in-flight task 注册表(lifespan drain 用)
_recovery_lock: asyncio.Lock # 跨 worker 会话恢复的并发锁
```
⚠️ **关键限制**:这些都是 per-process 内存。多 worker 部署下,worker B 看不到 worker A 的状态 → SSE 事件分发失败。详见 `frontend-web/docs/ai-writing-说明.md` 第五、六章。
### `_stream_graph_segment`(graph 推进函数)
`/start` 和 `/resume` 都调用这个函数(用 `_spawn_graph_task` 包装),负责:
1. 调 `graph.astream(input_or_command, config=config, stream_mode="updates")`
2. 把每个 chunk 解析成 SSE 事件 put 进 `_event_queues[session_id]`
3. 遇到 `__interrupt__` → 推 `await_user` 事件 → 返回(不关闭 queue)
4. 遇到 END → 推 `done` 事件 → 关闭 queue(put None 哨兵)
5. 任何异常 → 推 `error` 事件 → 关闭 queue
6. finally 段:更新 DB 最终状态、清理所有模块字典
### SSE 生成器 `_sse_generator`
`GET /stream` 的实现,从 `_event_queues[session_id]` 拿事件 → `_sse(event, data)` 序列化 → yield 给 fastapi `StreamingResponse`。
**关键逻辑**:
- 队列不存在时**等待 5 秒**,期间触发 `_recover_session`(跨 worker / 进程重启恢复)
- `_sse()` 三层兜底:`default=str` → 手写 notice 帧 → 永不抛 TypeError
- 收到 None 哨兵 → 关闭流
### `_recover_session`(跨 worker 恢复)
当 worker 内存里没有 session_id 但 sqlite checkpointer 有快照时:
1. 从 checkpointer 读出 state snapshot
2. 重建 `_event_queues[sid]` 队列
3. 从 snapshot 提取当前 interrupt 负载,put 进队列
4. 同时尽量恢复 `_session_repos`、`_session_user_ids` 等
返回三种状态:
- `"interrupted"` — 图停在干预点,已重建队列
- `"finished"` — 图已完成或中断在节点中间,推 done/error 事件后关闭
- `"unrecoverable"` — sqlite 也没有这个 thread_id → 真的丢了
### `_spawn_graph_task` / `wait_for_inflight_graph_tasks`(优雅关闭)
- `_spawn_graph_task(coro)` — 启 task 并自动登记到 `_inflight_graph_tasks`,完成时反登记
- `wait_for_inflight_graph_tasks(timeout)` — lifespan shutdown 等所有 task 跑到下一个 interrupt / END(默认 30s),超时则 cancel 强制结束
---
## 六、Graph 与持久化(graph.py)
### Lazy 单例 + 自动降级
```python
AI_WRITING_GRAPH = None # 模块级槽位(测试可 monkeypatch)
_async_checkpointer_singleton = None # checkpointer 单例
async def get_ai_writing_graph():
"""懒加载 graph。第一次调用时按 config.yaml 决定 checkpointer。"""
global AI_WRITING_GRAPH
if AI_WRITING_GRAPH is not None:
return AI_WRITING_GRAPH
checkpointer = await _get_async_checkpointer()
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
return AI_WRITING_GRAPH
```
### Checkpointer 优先级(_get_async_checkpointer)
```
1. config.yaml 的 checkpointer 段
├── type: sqlite → AsyncSqliteSaver(aiosqlite.connect(...))
├── type: postgres → AsyncPostgresSaver(AsyncConnectionPool(...))
└── type: memory → InMemorySaver
2. 任何初始化失败 → InMemorySaver(降级,打 warning)
3. 未配置 checkpointer 段 → InMemorySaver(降级,打 warning)
4. 未知 type(如 "mysql") → InMemorySaver(降级,打 warning)
```
**核心契约**:`_get_async_checkpointer()` **绝不抛异常**,永远返回一个可用的 saver。这是「写作流程能完整继续」契约的关键。
### Shutdown
`shutdown_ai_writing_graph()` 在 lifespan shutdown 时调用:
- 关闭 `aiosqlite.Connection` / `AsyncConnectionPool`
- 清空 `AI_WRITING_GRAPH` / `_async_checkpointer_singleton`
调用顺序(详见 `app/gateway/app.py` lifespan):
1. 停清理任务
2. `wait_for_inflight_graph_tasks(30s)`
3. `shutdown_ai_writing_graph()`
---
## 七、节点详解(nodes/)
### `nodes/_utils.py`(公共工具)
| 函数 / 类 | 用途 |
|---|---|
| `get_model(config, model_name)` | 从 config.configurable 拿 model_name 或回退默认 |
| `parse_json(text, fallback)` | 鲁棒 JSON 解析(支持代码块、不完整 JSON、转义错误) |
| `_extract_balanced(text)` | 找出第一对平衡的 `{...}` |
| `_repair_json(text)` | 修复常见的 JSON 错误(缺尾引号等) |
| `ThinkTagParser` | 流式解析 `<think>...</think>` 标签(思考 / 正文分流) |
| `llm_json(system, user, fallback, config)` | 调 LLM 并解析 JSON 输出 |
| `count_words(text)` | 中文 + 英文混合计字数 |
### `nodes/intent_parser.py` — 意图解析
- 调 LLM 把用户意图(`user_intent` 字符串)解析成结构化字段:`article_type`、`target_audience`、`word_count_target`
- 用户已经在 `AIWritingRequest` 里指定的字段会**覆盖** LLM 推断
- 输出:更新 state 的对应字段,推 `step_done` 事件
### `nodes/researcher.py` — 素材检索
- 根据 `user_intent` 用 LLM 生成 N 个关键词(N = `keyword_count`,默认 6)
- 用 `search/*` 的 provider 并行检索
- 流式推 `research_progress` 事件("正在搜索关键词 X 中...")
- 输出:`material_package = {keywords, materials, summary}`,推 `materials_ready`
### `nodes/writer_outline.py` — 大纲规划
- 输入:`user_intent` + `material_package`
- 调 LLM 流式生成大纲 JSON,期间推 `outline_chunk`(正文 + thinking 分流)
- 解析最终 JSON → `current_outline`,推 `outline_ready`
### `nodes/writer_draft.py` — 草稿生成(最复杂)
- 按大纲**逐章节**调 LLM 生成正文,每章节独立流式推 `draft_chunk`
- 检测 LLM 是否「拒答」(`_looks_like_refusal`),如果是则换提示重试
- **严格模式下**:如果章节关联的素材不足,把该章节加入 `blocked_sections`,触发 `pause_section_help`
- 完成所有章节后拼成 `current_draft.full_markdown`,推 `draft_ready`
### `nodes/editor.py` — 编辑审核(3 维度并行)
- 3 个维度(**fact / logic / language**)并行调 LLM
- 每个维度独立流式推 `review_chunk`,完成后推 `review_item_done`
- 三个都完成后聚合成 `review_result`,推 `review_ready`
- 评分加权(fact 40% + logic 30% + language 30%),超过 `pass_threshold`(默认 80)verdict = `pass`
---
## 八、持久化(persistence/ai_writing_sessions/)
### `model.py` — ORM
```python
class AIWritingSessionRow(Base):
__tablename__ = "ai_writing_sessions"
id: Mapped[str] # = session_id(uuid)
user_id: Mapped[str | None] # NULL = no-auth 模式
title: Mapped[str] # 从 user_intent 截取前 50 字
user_intent: Mapped[str | None]
status: Mapped[str] # in_progress / draft_ready / review_ready / done / error
draft_title: Mapped[str | None]
draft_markdown: Mapped[str | None]
completed_interventions: Mapped[str | None] # JSON 字符串
review_result: Mapped[str | None] # JSON 字符串
transcript: Mapped[str | None] # PortableLongText(MySQL→LONGTEXT,4GB)
created_at / updated_at: BeijingDateTime
```
### `sql.py` — Repository
| 方法 | 用途 |
|---|---|
| `create(id, title, user_intent, user_id)` | 启动写作时建一行 |
| `get(session_id, user_id)` | 取单个(带 user_id 权限校验) |
| `list(user_id, limit)` | 列出当前用户的会话 |
| `update(session_id, user_id, ...)` | 更新指定字段(其他保持不变) |
| `delete(session_id, user_id)` | 删除(带权限校验) |
| `list_older_than(cutoff, only_finished, limit)` | 清理用:列出过期 session_id |
| `delete_by_ids(session_ids)` | 清理用:批量删除 |
### Alembic 迁移
- `20260517_03_ai_writing_transcript.py` — 加 transcript 列(TEXT)
- `20260519_02_ai_writing_transcript_longtext.py` — MySQL 上把 transcript 改成 LONGTEXT(修 64KB 限制导致的 500)
---
## 九、定期清理(ai_writing_cleanup.py)
### 设计目标
- 删超过 `retention_days`(默认 7)的旧会话
- 联动清理:MySQL 业务行 + SQLite checkpointer 里的 thread 状态
- **绝对不删正在写的会话**(活跃保护)
### 关键函数
```python
async def run_cleanup_once(*, retention_days, only_finished=False, batch_size=500) -> dict
```
返回 `{deleted_db, deleted_checkpoints, skipped_active}`。
执行流程:
1. 调 `repo.list_older_than(cutoff, only_finished, batch_size)` 拿过期 id
2. 从 `_event_queues.keys()` 拿活跃 session_id 集合 → **过滤掉所有仍在跑的**
3. 对每个待删 id:调 `checkpointer.adelete_thread(id)` 删 checkpoint
4. 调 `repo.delete_by_ids(ids)` 批量删 MySQL
5. 循环直到本批 < batch_size
### Cron 调度
`cleanup_scheduler_loop()` 常驻后台任务:
- 用 `croniter` 解析 `ai_writing.cleanup.cron`(默认 `"0 3 * * *"`,每天凌晨 3 点)
- `asyncio.sleep` 到下次触发时间
- 多 worker 用文件锁(`runtime_home() / ".ai_writing_cleanup.lock"`)做 leader election
### Admin 手动触发
`POST /api/ai-writing/admin/cleanup`:
- 鉴权:`system_role == "admin"`
- 可选覆盖 `retention_days` / `only_finished` / `batch_size`
- 即使传 `retention_days: 0` 也不会秒杀活跃会话(活跃保护硬保证)
---
## 十、配置(config.yaml)
```yaml
# Checkpointer:graph 状态持久化
checkpointer:
type: sqlite # sqlite | postgres | memory
connection_string: .deer-flow/data/checkpoints.db
# 清理调度
ai_writing:
cleanup:
enabled: true
retention_days: 7
cron: "0 3 * * *" # 每天凌晨 3 点
delete_only_finished: false
batch_size: 500
# (可选)素材检索 provider 配置
search:
provider: deerflow.agents.ai_writing.search.internet_search:WebSearchProvider
...
```
`_ai_writing_config()` 函数从 `get_app_config().ai_writing` 里读 dict,节点通过 `config.configurable.ai_writing_config` 拿到。
---
## 十一、SSE 事件协议(后端 → 前端契约)
后端通过 `_sse(event_type, data)` 推 `event: <type>\ndata: <json>\n\n` 格式。
### 推出事件的位置
| 事件 | 推出位置 |
|---|---|
| `connected` | `_sse_generator` 入口 |
| `step_start` / `step_done` | 节点内部主动 emit |
| `research_progress` | `researcher_node` 内的 `_emit` |
| `materials_ready` | `_stream_graph_segment` 检测到 `material_package` 字段 |
| `outline_chunk` | `writer_outline_node` 流式推(通过 `sse_queue`) |
| `outline_ready` | `_stream_graph_segment` 检测到 `current_outline` |
| `draft_chunk` | `writer_draft_node` 按章节流式推 |
| `draft_ready` | `_stream_graph_segment` 检测到 `current_draft` |
| `review_chunk` / `review_item_done` | `editor_node` 3 维度并行流式 |
| `review_ready` | `_stream_graph_segment` 检测到 `review_result` |
| `await_user` | `_stream_graph_segment` 捕获 `__interrupt__` |
| `resumed` | pause 节点 return 时塞进 `progress_events` |
| `notice` | 节点内手动推(非致命告警) |
| `done` | `_stream_graph_segment` 正常结束 |
| `error` | `_stream_graph_segment` except 分支 |
### 节点如何流式推 chunk
通过 `config.configurable.sse_queue`:
```python
async def writer_outline_node(state, config):
sse_queue = config.get("configurable", {}).get("sse_queue")
sse_session_id = config.get("configurable", {}).get("sse_session_id")
async for chunk in llm.astream(prompt):
if sse_queue:
await sse_queue.put(("outline_chunk", {
"session_id": sse_session_id,
"text": chunk.content,
"is_thinking": False,
}))
return {"current_outline": parsed, ...}
```
`sse_queue` 是路由层在 `_make_config` 里注入的 `_event_queues[session_id]`。
---
## 十二、扩展指南:常见开发任务
### 任务 1:新增一个 Graph 节点
**例**:在 `editor` 之后加一个「术语统一」节点。
1. **写节点**:在 `nodes/` 新建 `terminology.py`:
```python
async def terminology_node(state, config):
# ... 读 current_draft 改术语
return {
"current_draft": new_draft,
"progress_events": [{"type": "step_done", "agent_name": "术语统一", "message": "..."}],
}
```
2. **注册到 graph**:在 `graph.py` 的 `build_ai_writing_graph` 里:
```python
from .nodes.terminology import terminology_node
g.add_node("terminology", terminology_node)
# 改 editor 后的边
g.add_edge("editor", "terminology")
g.add_edge("terminology", "pause_review")
```
3. **State 加字段**(如果需要):`state.py` 加新字段到 `AIWritingState`
4. **前端事件支持**(如果要新事件类型):见前端文档「扩展指南」
5. **测试**:在 `tests/` 加单元测试,至少测节点能正常 return 和 graph 拓扑能编译
### 任务 2:新增一个用户干预点
1. **写 pause 节点**:在 `graph.py` 顶部加:
```python
async def pause_xxx(state):
user_input = interrupt({
"pause_point": "xxx_confirm",
"message": "...",
"context_field": state["..."],
})
action = user_input.get("action", "confirm")
if action == "confirm":
return {"status": "next_status", "progress_events": [{"type": "resumed", ...}]}
# ... 其他分支
```
2. **路由层 ResumeRequest 加字段**(如果干预需要新字段):
```python
class ResumeRequest(BaseModel):
# ...
new_field: Optional[str] = None
```
3. **`resume_writing` 路由把字段塞进 Command(resume=...)**:
```python
intervention = {
# ...
"newField": body.new_field,
}
```
4. **graph 拓扑加 pause 节点的边**:在 build_ai_writing_graph 里
5. **前端配合**:见前端文档
### 任务 3:新增一种素材检索 provider
1. 在 `agents/ai_writing/search/` 加新文件,实现 `SearchProvider` 接口(参考 `internet_search.py`):
```python
class MyProvider(SearchProvider):
async def search(self, keyword: str, limit: int) -> list[SearchResult]:
...
```
2. 配置 `config.yaml`:
```yaml
ai_writing:
search:
provider: my_pkg.search:MyProvider
# provider 特定的配置...
```
3. `researcher_node` 通过 `config.configurable.ai_writing_config.search.provider` 用 `resolve_class` 动态加载
### 任务 4:修改 LLM Prompt
所有 prompt 集中在 `agents/ai_writing/prompts/`:
- `researcher_prompts.py` — 检索专家
- `writer_prompts.py` — 大纲 + 草稿
- `editor_prompts.py` — 审核
改 prompt 后**强烈建议**:
1. 在 `tests/` 加针对该 prompt 输出的回归测试(用 LLM mock 或固定输入)
2. 跑现有测试确认拓扑兼容
### 任务 5:扩展持久化字段
例:给 `ai_writing_sessions` 加一个 `total_tokens` 列。
1. `model.py` 加列:
```python
total_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
```
2. **必须**新建 alembic 迁移:
```bash
PYTHONPATH=. uv run alembic revision -m "add total_tokens column"
```
然后填 upgrade/downgrade 逻辑(参考 `20260519_02_ai_writing_transcript_longtext.py`)
3. `sql.py` 的 `update()` / `to_dict()` 加字段处理
4. `routers/ai_writing.py` 的 `SessionResponse` 加字段
5. 测试:在 `tests/test_ai_writing_session_repo.py` 加新字段的 round-trip 测试
---
## 十三、容错策略(关键设计原则)
### 「写作流程能完整继续」契约
后端承诺:**除非 Pydantic 校验失败(4xx),AI 写作的 4 个核心接口任何后端组件失败都不阻断当次写作的完整执行**。
### 落实位置
| 失败点 | 兜底位置 | 行为 |
|---|---|---|
| sqlite checkpointer 初始化失败 | `_get_async_checkpointer` | 自动降级 InMemorySaver |
| SSE JSON 序列化失败 | `_sse()` | 三层兜底(default=str → notice 帧) |
| DB `create()` session 行失败 | `start_writing` | try/except 吞掉,session_id 仍返回 |
| DB `update(draft)` 失败 | `_stream_graph_segment` | try/except 吞掉,graph 继续推 |
| DB `update(review)` 失败 | 同上 | 同上 |
| DB `update(transcript)` 失败 | `save_transcript` 路由 | 返回 `success=false`,不抛 5xx |
| 文章类型加载失败 | `start_writing` | 退回 `[]` |
| Graph 编译失败 | `start_writing` | 抛 503(极端情况,节点 import 错) |
| Graph 运行时节点抛错 | `_stream_graph_segment` | except 推 error 事件,session 结束 |
| Cleanup 误删活跃会话 | `run_cleanup_once` | 活跃保护硬过滤 |
### 容器优雅关闭
lifespan shutdown 顺序:
1. 停 cleanup scheduler
2. `wait_for_inflight_graph_tasks(30s)` — 等所有 in-flight 写作跑到下一个 checkpoint
3. 超时未结束的 task `cancel()` + 2s ack 等待
4. `shutdown_ai_writing_graph()` — 关闭 aiosqlite Connection / pg pool
---
## 十四、测试覆盖
| 测试文件 | 数量 | 覆盖范围 |
|---|---|---|
| `tests/test_ai_writing_session_repo.py` | 7 | Repo CRUD + 清理用的 list_older_than / delete_by_ids |
| `tests/test_ai_writing_session_recovery.py` | 8 | `_recover_session` / `_extract_interrupt` / `resume_writing` 跨 worker 恢复 |
| `tests/test_ai_writing_transcript_endpoint.py` | 5 | transcript 保存兜底 |
| `tests/test_ai_writing_cleanup.py` | 26 | 清理 + drain + checkpointer 降级 + SSE 序列化 + admin 接口 + 多 worker 检测 |
| `tests/test_portable_long_text.py` | 5 | LONGTEXT 列类型 + 迁移链 |
跑全部:
```bash
PYTHONPATH=. uv run pytest tests/test_ai_writing*.py tests/test_portable_long_text.py -v
```
期望 **51 个测试全过**。
---
## 十五、关键调试 / 排查命令
### 看 graph 当前状态
```python
# 进 python shell
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
graph = await get_ai_writing_graph()
snap = await graph.aget_state({"configurable": {"thread_id": "<session_id>"}})
print(snap.values, snap.next, snap.tasks)
```
### 直接读 sqlite checkpointer
```bash
sqlite3 /app/.deer-flow/data/checkpoints.db
.tables # 看表(checkpoints / checkpoint_writes 等)
SELECT thread_id, checkpoint_ns, checkpoint_id FROM checkpoints
WHERE thread_id = '<session_id>' LIMIT 5;
```
### 查 MySQL 业务行
```sql
SELECT id, status, draft_title, updated_at,
JSON_LENGTH(transcript) AS transcript_size
FROM ai_writing_sessions
WHERE id = '<session_id>';
```
### 查内存中的活跃会话(调试时手动加)
```python
# 临时在路由加一个 admin 接口
@router.get("/admin/inflight")
async def admin_inflight():
return {
"event_queues": list(_event_queues.keys()),
"inflight_tasks": len(_inflight_graph_tasks),
"pending_pause": list(_session_pending_pause.keys()),
}
```
### 日志关键字
```bash
docker logs <gateway> 2>&1 | grep -E 'ai_writing|AsyncSqliteSaver|checkpointer|recovered'
```
期望看到(启动时):
- `ai_writing: 使用 AsyncSqliteSaver (xxx.db)` — 持久化生效
- `ai_writing_cleanup: 已启动 cron=...` — 清理调度起来
警告:
- `ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver` — sqlite 故障
- `⚠ 检测到 WEB_CONCURRENCY=N` — 多 worker 警告(详见 `ai-writing-说明.md`)
---
## 十六、性能与限制
### 单 session 资源占用
- 内存:~50KB(state + progress_events 累计)
- sqlite checkpoint:每次 state update 写一行(一篇完整文章约 20-50 行)
- LLM tokens:约 30K-100K(取决于素材量 + 字数 + 修订次数)
### 并发上限
- 单 worker 内:理论上百级 session 并发(IO bound),实际 LLM API rate limit 才是瓶颈
- 多 worker:**当前 in-memory dict 隔离,详见 ai-writing-说明.md 第六章**
### 长会话风险
- progress_events 累加无界 → 长会话 state 可能膨胀到几 MB
- transcript JSON 同理 → MySQL 行可能上 MB(但 LONGTEXT 列上限 4GB,影响读取性能)
### 优化方向
- progress_events 节流(聚合 chunk 事件)
- transcript 增量保存(diff 而非全量覆盖)
- 节点级 LLM 调用加 retry(tenacity)
---
## 十七、与主 Agent 的关系
| 维度 | 主 Agent | AI 写作 |
|---|---|---|
| Graph 入口 | `make_lead_agent` | `AI_WRITING_GRAPH` |
| Middleware | 18 层中间件 | 无(直接节点链) |
| State | `ThreadState` | `AIWritingState` |
| Checkpointer | `make_checkpointer()`(共享) | `_get_async_checkpointer()`(同一文件,不同 thread_id 命名空间) |
| 路由前缀 | `/api/langgraph/*` `/api/threads/*` | `/api/ai-writing/*` |
| 流式协议 | LangGraph stream_mode(messages-tuple/values) | 自定义 SSE 事件 |
| 用户干预 | 通过 `ask_clarification` 工具 | 通过 LangGraph `interrupt()` |
| 持久化业务表 | `threads_meta`、`runs` | `ai_writing_sessions` |
二者**完全独立**,仅在「共享同一个 sqlite checkpointer 文件」这一点上有间接关联(也可以拆开,性能层面没区别)。
---
## 十八、未来 RFC 方向
| 主题 | 描述 |
|---|---|
| 多 worker 部署支持 | 把 `_event_queues` 等搬 Redis Pub/Sub,参考 `ai-writing-说明.md` 方案 B |
| Graph 拓扑可配置 | 现在节点链路写死,未来按 article_type 走不同拓扑(如「研报」vs「散文」) |
| 节点级重试 | LLM 抽风时自动重试 N 次,参考 `tenacity` |
| 素材跨 session 复用 | 同一用户多次写作可能用同一批素材 → 缓存层 |
| 多模态素材 | 支持图片素材输入和图文混合输出 |
| 中文标点 / 排版规则注入 | 在节点 prompt 里强化中文出版规范 |
---
## 附录 A:相关文件速查
| 文件 | 行数 | 用途 |
|---|---|---|
| `routers/ai_writing.py` | 1006 | 路由 + 状态字典 + SSE + drain |
| `agents/ai_writing/graph.py` | 538 | StateGraph + checkpointer 工厂 |
| `agents/ai_writing/state.py` | 121 | AIWritingState 定义 |
| `agents/ai_writing/nodes/_utils.py` | 242 | LLM 工具 |
| `agents/ai_writing/nodes/intent_parser.py` | 131 | 节点:意图解析 |
| `agents/ai_writing/nodes/researcher.py` | 271 | 节点:素材检索 |
| `agents/ai_writing/nodes/writer_outline.py` | 147 | 节点:大纲规划 |
| `agents/ai_writing/nodes/writer_draft.py` | 318 | 节点:草稿生成 |
| `agents/ai_writing/nodes/editor.py` | 180 | 节点:编辑审核 |
| `app/gateway/ai_writing_cleanup.py` | ~280 | 清理 + 调度 |
| `persistence/ai_writing_sessions/sql.py` | ~200 | Repository |
| `persistence/ai_writing_sessions/model.py` | ~35 | ORM |
---
## 附录 B:常见踩坑
| 现象 | 根因 | 解决 |
|---|---|---|
| `/start` 报 500 | graph 编译失败(多半是 import 错) | 看启动日志 stack trace |
| `/resume` 报 404 | session_id 不存在或 checkpointer 没快照 | 看 `ai_writing recover` 日志 |
| transcript 500 | DB 列容量 / 网络抖 | ✅ 已修,前端 fire-and-forget |
| SSE 卡住不动 | 多 worker SSE 隔离 / 反代超时 | 见 `ai-writing-说明.md` |
| `draft_chunk` 丢失内容 | `ThinkTagParser` 在 `<think>` 标签边界处理 | 看 `_utils.py:ThinkTagParser.feed` 单测 |
| graph 卡在某节点不退出 | LLM API 卡住 / 无限重试 | 看节点日志,加 timeout |
| cleanup 没跑 | cron 表达式配错 / 文件锁被其他 worker 持有 | 看 `ai_writing_cleanup` 日志 |
| 草稿在前端看不到但 MySQL 有 | 多 worker 导致 SSE 事件丢失 | 同上 |
| 配 mysql 做 checkpointer 报错 | LangGraph 不支持 MySQL checkpointer | 用 sqlite 或 postgres |
---
如需补充内容,请直接编辑本文件,并在 git commit message 里注明改动原因。
@@ -0,0 +1,278 @@
# 公开定时任务 API
本文档描述用于对接外部系统的**公开只读**定时任务接口,以及配套的公开网页入口。所有 `/api/public/*` 接口都**无需登录**,可以直接调用。
> ⚠️ 任何接入这套接口的系统需要确认:暴露 task / run 列表与正文是符合贵方业务安全要求的。如果不希望全员可见,请走带鉴权的 `/api/scheduled-tasks/*`。
---
## 1. 基础信息
| 项 | 值 |
|---|---|
| 网关基址(开发) | `http://localhost:8001` |
| 网关基址(生产) | 部署到 Nginx 后通常是 `https://<你的域名>` |
| 公开 API 前缀 | `/api/public/scheduled-tasks` |
| 鉴权 | **无需**鉴权(白名单在 `auth_middleware._PUBLIC_PATH_PREFIXES`) |
| 编码 | 请求 / 响应均为 UTF-8,响应类型 `application/json`(下载除外) |
| 错误响应 | 标准 FastAPI 错误体 `{"detail": "..."}`;HTTP 状态码 `404 / 400 / 500` 等 |
公开网页页面:
```
#/public/scheduled-tasks/<task_id>
```
例如 `https://<你的域名>/#/public/scheduled-tasks/8f1d8b9c-...`。
左侧列出该任务所有执行记录,默认打开第一条;右侧默认渲染 Markdown 正文,点击 Tab 可以查看“结果文件”或“元数据”,点击左侧任意记录右侧内容会跟随切换。
---
## 2. 接口列表
### 2.1 列出全部定时任务
```
GET /api/public/scheduled-tasks
```
**Query 参数**
| 名称 | 类型 | 必填 | 默认 | 说明 |
|---|---|---|---|---|
| `search` | string | 否 | — | 任务名模糊匹配(不区分大小写) |
| `limit` | int | 否 | 200 | 单页数量,1–500 |
| `offset` | int | 否 | 0 | 偏移量,配合 `limit` 分页 |
**响应**
```json
{
"total": 12,
"limit": 200,
"offset": 0,
"tasks": [
{
"task_id": "8f1d8b9c-...",
"name": "每日舆情简报",
"schedule_text": "每天 09:00",
"cron_expr": "0 9 * * *",
"timezone": "Asia/Shanghai",
"enabled": true,
"next_run_at": "2026-05-21T01:00:00+00:00",
"last_run_at": "2026-05-20T01:00:12+00:00",
"last_status": "succeeded",
"published": true
}
]
}
```
> 已脱敏:`user_id`、`creator_email`、`prompt`、`scheduler_thread_id`、`execution_context` 等私有字段不会出现在公开响应里。
### 2.2 获取单个任务元数据
```
GET /api/public/scheduled-tasks/{task_id}
```
**响应**:与上面 `tasks[]` 的元素结构一致;任务不存在返回 `404`。
### 2.3 列出某任务的所有执行记录
```
GET /api/public/scheduled-tasks/{task_id}/runs
```
**Query 参数**
| 名称 | 类型 | 必填 | 默认 | 说明 |
|---|---|---|---|---|
| `limit` | int | 否 | 50 | 最近执行数量,1–200 |
**响应**
```json
{
"task": { "task_id": "...", "name": "...", "schedule_text": "...", "timezone": "...", "enabled": true },
"total": 7,
"runs": [
{
"id": "run-uuid-...",
"task_id": "8f1d8b9c-...",
"scheduled_for": "2026-05-20T01:00:00+00:00",
"started_at": "2026-05-20T01:00:01+00:00",
"finished_at": "2026-05-20T01:00:12+00:00",
"status": "succeeded",
"error": null,
"retry_count": 0,
"result": {
"title": "2026-05-20 舆情简报",
"content": "# 简报\n\n...",
"format": "markdown",
"files": [
{ "index": 0, "name": "report.md", "mime_type": "text/markdown", "kind": "markdown", "size": 4321 }
],
"created_at": "2026-05-20T01:00:12+00:00"
}
}
]
}
```
> 已脱敏:返回的 `files[]` 不包含服务器绝对路径;客户端通过下标 `index` 调用文件接口。
### 2.4 获取单条执行记录
```
GET /api/public/scheduled-tasks/runs/{run_id}
```
返回单条 run 数据(结构同 2.3 中的 `runs[]` 元素)。
### 2.5 获取单个结果文件的内容(可预览文本直接内联)
```
GET /api/public/scheduled-tasks/runs/{run_id}/files/{file_index}
```
`file_index` 来自 `run.result.files[].index`。
**响应**
```json
{
"index": 0,
"name": "report.md",
"mime_type": "text/markdown",
"kind": "markdown",
"size": 4321,
"previewable": true,
"content": "# 简报\n\n..."
}
```
二进制文件 `previewable=false`、`content=null`,请走 2.6 下载。
### 2.6 下载单个结果文件
```
GET /api/public/scheduled-tasks/runs/{run_id}/files/{file_index}/download
```
返回原始文件二进制流,附带 `Content-Disposition: attachment` 头部。
### 2.7 打包下载整个执行结果
```
GET /api/public/scheduled-tasks/runs/{run_id}/download
```
- 如果该 run **没有附加文件**:直接返回 `result.md`(UTF-8 BOM)。
- 如果有 ≥ 1 个附加文件:返回 ZIP,里面包含:
- `result.md` — 标题 + 正文 Markdown
- 每个生成文件原样
---
## 3. 调用示例
### curl
```bash
# 列出全部任务
curl -s "https://<domain>/api/public/scheduled-tasks?limit=50"
# 查看任务的执行记录
curl -s "https://<domain>/api/public/scheduled-tasks/<task_id>/runs"
# 下载某次执行
curl -L -o report.zip \
"https://<domain>/api/public/scheduled-tasks/runs/<run_id>/download"
```
### Python
```python
import requests
base = "https://<domain>/api/public/scheduled-tasks"
tasks = requests.get(f"{base}", params={"limit": 100, "search": "舆情"}).json()
for task in tasks["tasks"]:
runs = requests.get(f"{base}/{task['task_id']}/runs", params={"limit": 5}).json()
for run in runs["runs"]:
if run.get("result"):
print(task["name"], run["id"], run["result"]["title"])
print(run["result"]["content"][:200])
```
### 直接嵌入 iframe
```html
<iframe src="https://<domain>/#/public/scheduled-tasks/<task_id>"
style="width:100%;height:800px;border:0"></iframe>
```
---
## 4. 状态码
| 状态码 | 含义 |
|---|---|
| 200 | 成功 |
| 400 | 参数错误(如 `limit` 超出范围) |
| 404 | 任务 / 执行 / 文件 不存在 |
| 5xx | 网关 / 后台错误 |
---
## 5. 字段映射快查
### `task` 字段
| 字段 | 类型 | 说明 |
|---|---|---|
| `task_id` | string | 任务 ID |
| `name` | string | 任务名 |
| `schedule_text` | string | 人类可读的执行计划 |
| `cron_expr` | string | 规范化的 cron 表达式 |
| `timezone` | string | 时区 |
| `enabled` | bool | 是否启用 |
| `next_run_at` | string &#124; null | 下次执行时间(ISO 8601) |
| `last_run_at` | string &#124; null | 上次执行时间 |
| `last_status` | string &#124; null | 上次执行状态(`succeeded` / `failed` / `running` / `pending` / `…`) |
| `published` | bool | 是否被发布到广场 |
### `run` 字段
| 字段 | 类型 | 说明 |
|---|---|---|
| `id` | string | 执行 ID |
| `task_id` | string | 所属任务 ID |
| `scheduled_for` | string | 原定执行时间 |
| `started_at` | string &#124; null | 实际开始时间 |
| `finished_at` | string &#124; null | 完成 / 失败时间 |
| `status` | string | `pending` / `running` / `succeeded` / `failed` |
| `error` | string &#124; null | 失败描述 |
| `retry_count` | int &#124; null | 重试次数 |
| `result` | object &#124; null | 执行结果(见下) |
### `result` 字段
| 字段 | 类型 | 说明 |
|---|---|---|
| `title` | string | 结果标题 |
| `content` | string | 主体 Markdown / 文本 |
| `format` | string | `markdown` / `text` / `json` / `mixed` |
| `files[]` | array | 附加文件元数据,含 `index / name / mime_type / kind / size` |
| `created_at` | string | 结果生成时间 |
---
## 6. 实现位置
- 路由实现:[`app/gateway/routers/public_scheduled_tasks.py`](../app/gateway/routers/public_scheduled_tasks.py)
- 公开 URL 白名单:[`app/gateway/auth_middleware.py`](../app/gateway/auth_middleware.py) 中的 `_PUBLIC_PATH_PREFIXES`
- 前端公开页面:[`frontend-web/src/pages/PublicScheduledTaskPage.tsx`](../../../frontend-web/src/pages/PublicScheduledTaskPage.tsx)
- 路由注册:[`frontend-web/src/App.tsx`](../../../frontend-web/src/App.tsx) `/#/public/scheduled-tasks/:taskId`
@@ -7,7 +7,8 @@
"env": ".env",
"graphs": {
"lead_agent": "deerflow.agents:make_lead_agent",
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent"
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent",
"ai_writing": "deerflow.agents.ai_writing.graph:make_ai_writing_graph"
},
"auth": {
"path": "./app/gateway/langgraph_auth.py:auth"
@@ -1,3 +1,3 @@
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
from .graph import make_ai_writing_graph
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
__all__ = ["make_ai_writing_graph"]
@@ -1,7 +1,9 @@
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
from __future__ import annotations
from langgraph.checkpoint.memory import MemorySaver
import logging
from langchain_core.runnables import RunnableConfig
from langgraph.graph import END, StateGraph
from langgraph.types import interrupt
@@ -12,6 +14,32 @@ from .nodes.writer_outline import writer_outline_node
from .nodes.writer_draft import writer_draft_node
from .nodes.editor import editor_node
logger = logging.getLogger(__name__)
async def _mark_session_done(config: RunnableConfig | None) -> None:
"""把业务表行 status 写成 ``done``。
pause_draft_confirm action=finalize / pause_review_confirm force_finalize 时调,
让 LangGraph Server 路径下新会话有正确终态(老路径靠 ``_stream_graph_segment``
finally 块写终态,新路径完全不走它)。失败不影响 graph 流程。
"""
if not config:
return
thread_id = (config.get("configurable") or {}).get("thread_id")
if not thread_id:
return
from deerflow.persistence.ai_writing_sessions import get_default_store
from deerflow.runtime.user_context import AUTO
repo = get_default_store()
if repo is None:
return
try:
await repo.update(thread_id, user_id=AUTO, status="done")
except Exception:
logger.exception("ai_writing: mark_session_done failed (non-fatal) thread_id=%s", thread_id)
# ── 干预节点 ──────────────────────────────────────────────────────────────────
async def pause_material_confirm(state: AIWritingState) -> dict:
@@ -61,6 +89,26 @@ def _normalize_outline(outline: dict) -> dict:
return {"title": outline.get("title", ""), "sections": sections}
def _check_outline_material_support(outline: dict) -> str:
"""轻量启发式:统计未关联任何素材的章节,返回预警文案(空串=无需预警)。
仅做提示、不拦截流程;真正的素材不足会在写作节点被检出并触发求助暂停。
"""
sections = (outline or {}).get("sections", [])
weak = [
(s.get("section_title") or s.get("sectionTitle") or "")
for s in sections
if not (s.get("material_ids") or s.get("materialIds"))
]
weak = [t for t in weak if t]
if not weak:
return ""
return (
f"有 {len(weak)} 个章节未关联检索素材({('、'.join(weak))})。"
"严格依据素材模式下,这些章节在写作时可能因素材不足而暂停向你求助。"
)
async def pause_outline_confirm(state: AIWritingState) -> dict:
"""PAUSE-2:大纲确认"""
user_input: dict = interrupt({
@@ -74,23 +122,37 @@ async def pause_outline_confirm(state: AIWritingState) -> dict:
if action == "confirm":
raw_outline = user_input.get("editedOutline") or state["current_outline"]
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
events: list[dict] = [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}]
# 阶段四:严格模式下对大纲做轻量素材充分性预警(仅提示,不拦截流程)
if state.get("writing_mode") == "strict":
warning = _check_outline_material_support(edited)
if warning:
events.append({
"type": "notice", "agent_name": "系统",
"title": "素材充分性提示", "detail": warning,
})
return {
"current_outline": edited,
"last_user_intervention": user_input,
"status": "writing_draft",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}],
"progress_events": events,
}
# re_outline
feedback = user_input.get("outlineFeedback", "")
return {
updates: dict = {
"last_user_intervention": {**user_input, "_feedback": feedback},
"status": "writing_outline",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
}
# 用户在卡片里手动改过大纲 → 以手动版作为本次重规划的编辑基准
raw_edited = user_input.get("editedOutline")
if raw_edited:
updates["current_outline"] = _normalize_outline(raw_edited)
return updates
async def pause_draft_confirm(state: AIWritingState) -> dict:
async def pause_draft_confirm(state: AIWritingState, config: RunnableConfig) -> dict:
"""PAUSE-3:草稿确认"""
draft = state["current_draft"]
user_input: dict = interrupt({
@@ -104,6 +166,7 @@ async def pause_draft_confirm(state: AIWritingState) -> dict:
action = user_input.get("action", "to_editor")
if action == "finalize":
await _mark_session_done(config)
return {
"last_user_intervention": user_input,
"status": "done",
@@ -127,7 +190,66 @@ async def pause_draft_confirm(state: AIWritingState) -> dict:
}
async def pause_review_confirm(state: AIWritingState) -> dict:
async def pause_section_help(state: AIWritingState) -> dict:
"""素材不足求助点:严格模式下作家发现章节缺素材,让用户逐章决定如何处理。
用户对每个被阻塞章节选择:
- supplement:补充检索素材(整体回到检索流程重走)
- delete:删除该章节
- loose:按通用知识续写该章节
"""
blocked = state.get("blocked_sections") or []
user_input: dict = interrupt({
"pause_point": "section_help",
"message": "部分章节缺少足够素材支撑写作,请决定如何处理。",
"blocked_sections": blocked,
})
# sectionDecisions: [{"sectionTitle": "...", "decision": "supplement|delete|loose"}]
by_title: dict = {}
for d in user_input.get("sectionDecisions") or []:
title = d.get("sectionTitle") or d.get("section_title")
if title:
by_title[title] = d.get("decision", "loose")
# 任一章节选「补充检索」→ 整体回到检索,重走「检索 → 素材确认 → 大纲 → 写作」
supplement_reasons = [
b["reason"] for b in blocked if by_title.get(b["section_title"]) == "supplement"
]
if supplement_reasons:
return {
# supplement_search:补充检索(保留已有素材、追加新素材),完成后直接回写作、不重排大纲
"last_user_intervention": {**user_input, "action": "supplement_search", "_extra_keywords": supplement_reasons},
"status": "researching",
"blocked_sections": None,
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "补充检索素材中…"}],
}
# 否则:删除标记章节、记录「通用知识续写」章节,直接回写作
outline = state.get("current_outline") or {"title": "", "sections": []}
kept, loose_titles = [], []
for s in outline.get("sections", []):
title = s.get("section_title") or s.get("sectionTitle", "")
decision = by_title.get(title)
if decision == "delete":
continue
if decision == "loose":
loose_titles.append(title)
kept.append(s)
# 防御:用户把所有章节都删了 → 保留原大纲并全部按通用知识写,避免空文章
if not kept:
kept = outline.get("sections", [])
loose_titles = [s.get("section_title") or s.get("sectionTitle", "") for s in kept]
return {
"current_outline": {**outline, "sections": kept},
"last_user_intervention": {**user_input, "_loose_sections": loose_titles},
"status": "writing_draft",
"blocked_sections": None,
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "已按你的选择继续写作…"}],
}
async def pause_review_confirm(state: AIWritingState, config: RunnableConfig) -> dict:
"""PAUSE-4:编辑打回确认"""
user_input: dict = interrupt({
"pause_point": "review_confirm",
@@ -138,6 +260,7 @@ async def pause_review_confirm(state: AIWritingState) -> dict:
})
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
await _mark_session_done(config)
return {
"last_user_intervention": user_input,
"status": "done",
@@ -156,6 +279,11 @@ async def pause_review_confirm(state: AIWritingState) -> dict:
# ── 条件路由 ──────────────────────────────────────────────────────────────────
def after_researcher(state: AIWritingState) -> str:
"""检索完成后:补充检索(素材不足求助)直接回写作,否则去素材确认。"""
return "writer_draft" if state.get("status") == "writing_draft" else "pause_material"
def after_material_pause(state: AIWritingState) -> str:
return "researcher" if state["status"] == "researching" else "writer_outline"
@@ -164,6 +292,16 @@ def after_outline_pause(state: AIWritingState) -> str:
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
def after_writer_draft(state: AIWritingState) -> str:
"""写作完成后:素材不足则去求助暂停点,否则去草稿确认。"""
return "pause_section_help" if state["status"] == "awaiting_section_help" else "pause_draft"
def after_section_help_pause(state: AIWritingState) -> str:
"""求助暂停点之后:补充检索回 researcher,否则回写作。"""
return "researcher" if state["status"] == "researching" else "writer_draft"
def after_draft_pause(state: AIWritingState) -> str:
status = state["status"]
if status == "done":
@@ -188,44 +326,50 @@ def after_review_pause(state: AIWritingState) -> str:
return "end" if state["status"] == "done" else "writer_draft"
# ── 组装 Graph ────────────────────────────────────────────────────────────────
def build_ai_writing_graph():
# ── LangGraph Server 入口 ────────────────────────────────────────────────────
#
# compile 出裸 graph(不带 checkpointer),由 LangGraph Server 注入主 checkpointer
# (langgraph.json 里 ``make_checkpointer`` 配的那个)。AI 写作的 thread 状态与
# 主聊天共享同一份 saver —— LangGraph schema 用 ``(thread_id, checkpoint_ns)`` 做
# 主键,UUID 不会撞。
def make_ai_writing_graph(config=None):
"""LangGraph Server graph factory:不接收 checkpointer,由 Server 注入。
Args:
config: LangGraph Server 调用工厂时注入的 RunnableConfig(暂未消费)。
"""
g = StateGraph(AIWritingState)
g.add_node("intent_parser", intent_parser_node)
g.add_node("researcher", researcher_node)
g.add_node("pause_material", pause_material_confirm)
g.add_node("writer_outline", writer_outline_node)
g.add_node("pause_outline", pause_outline_confirm)
g.add_node("writer_draft", writer_draft_node)
g.add_node("pause_draft", pause_draft_confirm)
g.add_node("editor", editor_node)
g.add_node("pause_review", pause_review_confirm)
g.add_node("intent_parser", intent_parser_node)
g.add_node("researcher", researcher_node)
g.add_node("pause_material", pause_material_confirm)
g.add_node("writer_outline", writer_outline_node)
g.add_node("pause_outline", pause_outline_confirm)
g.add_node("writer_draft", writer_draft_node)
g.add_node("pause_section_help", pause_section_help)
g.add_node("pause_draft", pause_draft_confirm)
g.add_node("editor", editor_node)
g.add_node("pause_review", pause_review_confirm)
g.set_entry_point("intent_parser")
g.add_edge("intent_parser", "researcher")
g.add_edge("researcher", "pause_material")
g.add_edge("intent_parser", "researcher")
g.add_conditional_edges("researcher", after_researcher,
{"pause_material": "pause_material", "writer_draft": "writer_draft"})
g.add_conditional_edges("pause_material", after_material_pause,
{"researcher": "researcher", "writer_outline": "writer_outline"})
{"researcher": "researcher", "writer_outline": "writer_outline"})
g.add_edge("writer_outline", "pause_outline")
g.add_conditional_edges("pause_outline", after_outline_pause,
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
g.add_edge("writer_draft", "pause_draft")
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
g.add_conditional_edges("writer_draft", after_writer_draft,
{"pause_section_help": "pause_section_help", "pause_draft": "pause_draft"})
g.add_conditional_edges("pause_section_help", after_section_help_pause,
{"researcher": "researcher", "writer_draft": "writer_draft"})
g.add_conditional_edges("pause_draft", after_draft_pause,
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
g.add_conditional_edges("editor", after_review,
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
g.add_conditional_edges("pause_review", after_review_pause,
{"end": END, "writer_draft": "writer_draft"})
{"end": END, "writer_draft": "writer_draft"})
checkpointer = MemorySaver()
return g.compile(checkpointer=checkpointer)
AI_WRITING_GRAPH = build_ai_writing_graph()
return g.compile()
@@ -15,6 +15,54 @@ from langchain_core.runnables import RunnableConfig
logger = logging.getLogger(__name__)
def get_ai_writing_config() -> dict:
"""读 ``config.yaml`` 的 ``ai_writing`` 段。
阶段 7 之前由路由层 ``_make_config`` 把这份 dict 注入到 ``configurable.ai_writing_config``
给节点使用。阶段 7 删冗余后 ``_make_config`` 和注入路径全部消失(前端 ``stream.submit``
的 config 只传 ``model_name``)—— 节点改成自己直接从 ``get_app_config()`` 读,跨边界
更干净,``config.yaml`` 改完也不用重启前端 / 重发起会话。
返回字段(按使用方):
- researcher:``keyword_model`` / ``rank_model`` / ``max_materials`` /
``search_provider`` 等
- editor:``pass_threshold``
"""
try:
from deerflow.config import get_app_config
aw = getattr(get_app_config(), "ai_writing", None)
if isinstance(aw, dict):
return aw
return dict(aw) if aw else {}
except Exception:
logger.exception("ai_writing: 读取 app config 失败,使用空配置")
return {}
def safe_get_stream_writer():
"""返回一个 stream writer callable;在没有 LangGraph runtime 上下文时退化成 no-op。
AI 写作节点用它把流式 chunk 推到 LangGraph 原生 ``custom`` 通道。
场景:
- LangGraph Server 路径(``client.runs.stream`` 订阅了 ``stream_mode=["custom", ...]``)
→ 真实 writer,事件透传给 SDK 消费方。
- 老 ``/start /stream /resume`` 路径(``graph.astream(stream_mode="updates")``)
→ 真实 writer 仍可调用,只是没人订阅 custom 通道,事件被静默丢弃。
这是阶段 2「硬切到 get_stream_writer」的预期行为:老前端 EventSource 不再收到
token-level chunk,但整体进度卡(draft_ready / review_ready 这些 state 级事件)
仍正常 —— 阶段 3 前端切到 useStream 后立刻补齐。
- 单测里直接调节点函数(无 LangGraph runtime) → 返回 no-op,调用方零分支。
"""
try:
from langgraph.config import get_stream_writer # 延迟 import,便于单测打补丁
return get_stream_writer()
except Exception:
return lambda _payload: None
def get_model(config: RunnableConfig, model_name: str | None = None):
"""创建对话模型。
@@ -1,8 +1,8 @@
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
from __future__ import annotations
import asyncio
import logging
from typing import Callable
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
@@ -11,7 +11,7 @@ from ..state import AIWritingState, ReviewResult, ReviewIssue
from ..prompts.editor_prompts import (
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
from ._utils import get_ai_writing_config, get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
logger = logging.getLogger(__name__)
@@ -40,9 +40,9 @@ def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
async def _stream_one_review(
model, user_prompt: str, category: str,
sse_queue: asyncio.Queue | None, session_id: str,
writer: Callable[[dict], None],
) -> str:
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
"""流式执行单项审查,逐 chunk 经 LangGraph custom 通道推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
parser = ThinkTagParser()
async for chunk in model.astream([
@@ -55,16 +55,42 @@ async def _stream_one_review(
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("review_chunk", {
"session_id": session_id,
"category": category,
"text": text,
"is_thinking": is_thinking,
}))
writer({
"type": "review_chunk",
"category": category,
"text": text,
"is_thinking": is_thinking,
})
return full_text
async def _save_review_to_repo(config: RunnableConfig, review: ReviewResult) -> None:
"""把 review_result 异步落到业务表(best-effort,写失败不影响流程)。
``user_id`` 走 ``resolve_user_id(AUTO)`` 读 contextvar(与 intent_parser 建行、
writer_draft 写草稿同套规则)。
"""
cfg = config.get("configurable", {}) or {}
thread_id = cfg.get("thread_id")
if not thread_id:
return
from deerflow.persistence.ai_writing_sessions import get_default_store
from deerflow.runtime.user_context import AUTO
repo = get_default_store()
if repo is None:
return
try:
await repo.update(
thread_id,
user_id=AUTO,
status="review_ready",
review_result=dict(review),
)
except Exception:
logger.exception("editor: save_review failed (non-fatal) thread_id=%s", thread_id)
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
draft = state.get("current_draft")
if not draft:
@@ -78,9 +104,9 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
mats = _materials_summary(pkg)
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
writer = safe_get_stream_writer()
# 阶段 7 后从 app config 直接读,不走 configurable 注入
ai_writing_config = get_ai_writing_config()
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
REVIEW_TASKS = [
@@ -110,7 +136,7 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
for category, user_prompt in REVIEW_TASKS:
fallback = {"score": 80, "issues": []}
try:
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
full_text = await _stream_one_review(model, user_prompt, category, writer)
res = parse_json(full_text, fallback)
except Exception as e:
logger.warning("editor review [%s] failed: %s", category, e)
@@ -120,16 +146,13 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
results[category] = res
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
if sse_queue is not None:
issues = _parse_issues(res.get("issues", []), category)
await sse_queue.put(("review_item_done", {
"session_id": session_id,
"category": category,
"score": int(res.get("score", 80)),
"issues": [dict(iss) for iss in issues],
}))
else:
logger.debug("editor [%s] done (no sse_queue)", category)
issues = _parse_issues(res.get("issues", []), category)
writer({
"type": "review_item_done",
"category": category,
"score": int(res.get("score", 80)),
"issues": [dict(iss) for iss in issues],
})
# 汇总
fact_score = int(results["fact"].get("score", 80))
@@ -156,6 +179,9 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
revision_notes=all_issues,
)
# 阶段 2:节点内并存 DB 写入(路由层同样也写一遍,阶段 7 删冗余时移除路由那份)
await _save_review_to_repo(config, review)
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
notices: list[dict] = []
if review_errors:
@@ -1,12 +1,64 @@
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
from __future__ import annotations
import logging
import re
from langchain_core.runnables import RunnableConfig
from langchain_core.messages import HumanMessage, SystemMessage
from ..state import AIWritingState
from ._utils import llm_json
from ._utils import (
ThinkTagParser,
_content_to_text,
get_model,
parse_json,
safe_get_stream_writer,
)
logger = logging.getLogger(__name__)
def _truncate_title(text: str, max_len: int = 50) -> str:
"""与老 /start 路由里 _truncate_title 行为一致:UTF 字符截断 + 省略号。"""
text = (text or "").strip()
if len(text) <= max_len:
return text
return text[:max_len].rstrip() + "…"
async def _upsert_business_row(configurable: dict, state: AIWritingState) -> None:
"""在 intent_parser 入口幂等创建 ai_writing_sessions 业务表行。
LangGraph Server 路径下没有 /start 路由建行,必须节点自建,否则列表页查不到、
transcript / draft / review 后续写入都会因 update 找不到行而静默丢数据。
resume 路径下 thread_id 已经有行,upsert_session 只刷新 updated_at。
user_id 走 ``resolve_user_id(AUTO)``:FastAPI 请求线程把它写到 contextvar,
asyncio.create_task 继承父任务的 context,所以这里能拿到当前用户;
contextvar 没设时(测试 / 离线场景)抛 RuntimeError,try/except 兜底跳过。
"""
from deerflow.persistence.ai_writing_sessions import get_default_store
from deerflow.runtime.user_context import AUTO
thread_id = configurable.get("thread_id")
if not thread_id:
return
repo = get_default_store()
if repo is None:
return
user_intent = (state.get("user_intent") or "").strip()
try:
await repo.upsert_session(
id=thread_id,
title=_truncate_title(user_intent) or "AI 写作",
user_intent=user_intent,
user_id=AUTO,
status="in_progress",
)
except Exception:
logger.exception("intent_parser: upsert_session failed (non-fatal) thread_id=%s", thread_id)
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
_DEFAULT_ARTICLE_TYPES: list[dict] = [
@@ -79,6 +131,10 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
valid_keys: list[str] = [t["key"] for t in valid_article_types]
default_key: str = valid_keys[0] if valid_keys else "analysis"
# 业务表 upsert(LangGraph Server 路径下没有 /start 路由建行,节点入口自建;
# resume 时幂等不重复建。失败不阻断写作流程)。
await _upsert_business_row(configurable, state)
type_options = _build_type_options(valid_article_types)
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
user = _USER_TEMPLATE.format(
@@ -90,42 +146,79 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
if fallback_type not in valid_keys:
fallback_type = default_key
result = await llm_json(
system,
user,
fallback={
"article_type": fallback_type,
"target_audience": state.get("target_audience", "普通读者"),
"word_count_target": state.get("word_count_target", 800),
"refined_intent": state["user_intent"],
},
config=config,
)
fallback_result = {
"article_type": fallback_type,
"target_audience": state.get("target_audience", "普通读者"),
"word_count_target": state.get("word_count_target", 800),
"refined_intent": state["user_intent"],
}
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
# 流程仍可继续,但用户需知道结构化配置可能不准)。
# 流式调用:与 writer_outline / writer_draft 同套机制 —— LLM 逐字吐出 JSON,
# 通过 ``intent_chunk`` 推给前端打字机;完成后 ``parse_json`` 解析结构化结果,
# 再通过 ``intent_ready`` 推一份结构化字段给前端结果卡片。
writer = safe_get_stream_writer()
llm_error: str | None = None
full_text = ""
parser = ThinkTagParser()
try:
model = get_model(config)
async for chunk in model.astream(
[SystemMessage(content=system), HumanMessage(content=user)]
):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
writer({
"type": "intent_chunk",
"text": text,
"is_thinking": is_thinking,
})
except Exception as exc:
# 模型 / 网络挂掉 —— 不阻断流程,用 fallback 继续。
logger.error("intent_parser streaming failed: %s", exc, exc_info=True)
llm_error = f"{type(exc).__name__}: {exc}"
result = parse_json(full_text, fallback_result) if full_text else fallback_result
# 意图解析模型调用失败时以非致命 notice 提示用户(fallback 已兜底,流程仍可继续)
progress_events: list[dict] = []
if result.get("_llm_error"):
if llm_error:
progress_events.append({
"type": "notice",
"agent_name": "系统",
"title": "模型错误",
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
"detail": f"意图解析失败,已按默认配置兜底。原因:{llm_error}",
})
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
updates: dict = {"status": "researching", "progress_events": progress_events}
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
raw_type = result.get("article_type", "")
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
if result.get("target_audience"):
updates["target_audience"] = result["target_audience"]
final_article_type = raw_type if raw_type in valid_keys else fallback_type
final_target_audience = result.get("target_audience") or fallback_result["target_audience"]
word_count = _coerce_word_count(result.get("word_count_target"))
final_word_count = word_count if word_count is not None else fallback_result["word_count_target"]
final_refined_intent = result.get("refined_intent") or state["user_intent"]
# 推送结构化结果给前端 —— 结果卡片显示文章类型 / 目标读者 / 字数 / 精炼意图
writer({
"type": "intent_ready",
"article_type": final_article_type,
"target_audience": final_target_audience,
"word_count_target": final_word_count,
"refined_intent": final_refined_intent,
})
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
updates: dict = {"status": "researching", "progress_events": progress_events}
updates["article_type"] = final_article_type
if result.get("target_audience"):
updates["target_audience"] = final_target_audience
if word_count is not None:
updates["word_count_target"] = word_count
updates["word_count_target"] = final_word_count
if result.get("refined_intent"):
updates["user_intent"] = result["refined_intent"]
updates["user_intent"] = final_refined_intent
return updates
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import logging
import re
import uuid
from typing import List
@@ -14,11 +15,66 @@ from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
)
from ._utils import llm_json
from ._utils import get_ai_writing_config, llm_json, safe_get_stream_writer
logger = logging.getLogger(__name__)
_MAX_MATERIALS = 12
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 50
_DEFAULT_MAX_MATERIALS = 50
_MIN_MAX_MATERIALS = 1
_MAX_MAX_MATERIALS = 100
# 素材检索关键词数量边界:用户可指定 1~10 个,非法值回退默认 6
_DEFAULT_KEYWORD_COUNT = 6
_MAX_KEYWORD_COUNT = 10
def _resolve_keyword_count(raw: object) -> int:
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
try:
n = int(raw) # type: ignore[arg-type]
except (TypeError, ValueError):
return _DEFAULT_KEYWORD_COUNT
return max(1, min(_MAX_KEYWORD_COUNT, n))
def _resolve_max_materials(raw: object) -> int:
"""把 config.yaml 配置的素材总量上限规整到 [1, 100],非法值回退默认 20。"""
try:
n = int(raw) # type: ignore[arg-type]
except (TypeError, ValueError):
return _DEFAULT_MAX_MATERIALS
return max(_MIN_MAX_MATERIALS, min(_MAX_MAX_MATERIALS, n))
# 占位/示意性关键词(模型误抄提示词模板时产生),检索前剔除
_PLACEHOLDER_KEYWORD_RE = re.compile(r"^(关键词\d*|keyword\d*|kw\d*|\.+|…+)$", re.IGNORECASE)
def _clean_keywords(raw: object, fallback: str) -> list[str]:
"""清洗 LLM 产出的关键词:去空白、丢弃占位符(如「关键词1」「...」)、去重。
全部被滤掉时回退为写作意图本身,避免拿占位词检索出一堆无关素材。
"""
cleaned: list[str] = []
for kw in raw if isinstance(raw, list) else []:
s = str(kw).strip()
if not s or _PLACEHOLDER_KEYWORD_RE.match(s):
continue
cleaned.append(s)
deduped = list(dict.fromkeys(cleaned))
return deduped or [fallback]
def _per_keyword_limit(max_materials: int, total_keywords: int) -> int:
"""把素材总量上限均分到每个检索词上的检索条数(向上取整)。
向上取整保证合并去重后素材数仍能接近总量上限;最终会再统一截断到上限。
total_keywords <= 0 时回退为总量上限本身。
"""
if total_keywords <= 0:
return max_materials
return (max_materials + total_keywords - 1) // total_keywords
def _notice(title: str, detail: str, raw: str = "") -> dict:
@@ -37,52 +93,80 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
article_type = state.get("article_type", "analysis")
target_audience = state.get("target_audience", "普通读者")
# 检查是否是补充搜索(PAUSE-1 re_search action)
# 检查干预模式(PAUSE-1 素材确认 / 素材不足求助)
intervention = state.get("last_user_intervention") or {}
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
action = intervention.get("action")
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
# 素材不足求助点触发的「补充检索」:保留原大纲,检索完直接回写作、不走素材确认/大纲
is_supplement = action == "supplement_search"
# PAUSE-1 素材确认卡用户点「追加检索」并填了新关键词:
# - 跳过 LLM 重新生成检索词(节省一次调用 + 尊重用户意图)
# - 只检索 extra_keywords 这几个新词,不重检索之前 LLM 出过的词
# - 保留 existing material_package,按 URL 去重合并
is_append_only = action == "re_search" and bool(extra_keywords)
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
# 实时进度推送:阶段 2 起统一走 LangGraph 原生 custom 通道。
cfg = config.get("configurable", {})
sse_queue = cfg.get("sse_queue")
sse_session_id: str = cfg.get("sse_session_id", "")
writer = safe_get_stream_writer()
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
# 注:阶段 7 后这份配置由节点直接读 app config,不再走 configurable 注入。
ai_writing_config = get_ai_writing_config()
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
rank_model: str | None = ai_writing_config.get("rank_model") or None
# 素材总量上限(config.yaml: ai_writing.max_materials),按检索词数均分到每个词
max_materials = _resolve_max_materials(ai_writing_config.get("max_materials"))
async def _emit(message: str) -> None:
if sse_queue is None:
return
await sse_queue.put(("research_progress", {
"session_id": sse_session_id,
"message": message,
}))
def _emit(message: str) -> None:
writer({"type": "research_progress", "message": message})
# Step 1: 提取关键词
await _emit("正在生成检索词…")
kw_result = await llm_json(
EXTRACT_KEYWORDS_SYSTEM,
EXTRACT_KEYWORDS_USER.format(
user_intent=intent,
article_type=article_type,
target_audience=target_audience,
),
fallback={"keywords": [intent]},
config=config,
model_name=keyword_model,
)
keywords: list[str] = kw_result.get("keywords", [intent])
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
kw_result: dict = {} # 走追加分支时跳过 LLM 调用,但下方 notice 拼接需要兜底值
if is_append_only:
# 追加模式:直接用用户填的关键词,不再调 LLM
keywords: list[str] = _clean_keywords(extra_keywords, intent)
_emit(f"追加检索 {len(keywords)} 个关键词…")
else:
_emit("正在生成检索词…")
kw_result = await llm_json(
EXTRACT_KEYWORDS_SYSTEM,
EXTRACT_KEYWORDS_USER.format(
user_intent=intent,
article_type=article_type,
target_audience=target_audience,
keyword_count=keyword_count,
),
fallback={"keywords": [intent]},
config=config,
model_name=keyword_model,
)
# 清洗关键词:剔除占位符 / 空值,全空时回退写作意图
keywords = _clean_keywords(kw_result.get("keywords"), intent)
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
# Step 2: 并发搜索
provider = get_search_provider(ai_writing_config)
search_keywords = keywords[:6]
# 按用户指定的关键词数量截断。
# 追加模式:keywords 本身就是用户填的,全部检索不截断(用户填 N 个就搜 N 个)。
# 其他模式:受 keyword_count 限制,但 extra_keywords 在内时上限放宽到 keywords 总数。
if is_append_only:
search_keywords = keywords
else:
search_limit = keyword_count
if extra_keywords:
search_limit = max(search_limit, len(keywords))
search_keywords = keywords[:search_limit]
total = len(search_keywords)
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
per_keyword = _per_keyword_limit(max_materials, total)
_emit(f"检索词已生成,开始检索 {total} 个方向…")
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
completed = 0
@@ -90,22 +174,26 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
async def _search_one(kw: str) -> list[SearchResult]:
nonlocal completed
try:
return await provider.search(kw, max_results=3)
return await provider.search(kw, max_results=per_keyword)
finally:
completed += 1
if completed < total:
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
_emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
else:
await _emit(f"已完成全部 {total} 次检索")
_emit(f"已完成全部 {total} 次检索")
search_tasks = [_search_one(kw) for kw in search_keywords]
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
# 合并去重
# - 首次检索 / 普通 re_search(不带 extra_keywords,用户只想 LLM 重出):旧素材丢弃
# - 素材不足求助 supplement_search:保留旧素材 + 追加新检索结果
# - PAUSE-1 追加检索 is_append_only:保留旧素材 + 仅检索新关键词
seen_urls: set[str] = set()
raw_materials: list[Material] = []
existing_pkg = state.get("material_package")
if existing_pkg and intervention.get("action") != "re_search":
preserve_existing = existing_pkg and (is_supplement or is_append_only)
if preserve_existing:
for m in existing_pkg.get("materials", []):
seen_urls.add(m.get("url", ""))
raw_materials.append(m)
@@ -136,9 +224,9 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
search_errors.extend(getattr(provider, "errors", []))
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
await _emit("检索完成,正在筛选排序素材…")
_emit("检索完成,正在筛选排序素材…")
rank_error = ""
if len(raw_materials) > _MAX_MATERIALS:
if len(raw_materials) > max_materials:
materials_list = "\n".join(
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
for m in raw_materials
@@ -146,7 +234,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
rank_result = await llm_json(
RANK_MATERIALS_SYSTEM,
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
fallback={"ranked_ids": [m["id"] for m in raw_materials[:_MAX_MATERIALS]], "summary": f"共搜索到{len(raw_materials)}条素材"},
fallback={"ranked_ids": [m["id"] for m in raw_materials[:max_materials]], "summary": f"共搜索到{len(raw_materials)}条素材"},
config=config,
model_name=rank_model,
)
@@ -159,17 +247,23 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
for m in raw_materials:
if m["id"] not in sorted_ids_set:
sorted_materials.append(m)
materials = sorted_materials[:_MAX_MATERIALS]
materials = sorted_materials[:max_materials]
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
else:
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:_MAX_MATERIALS]
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:max_materials]
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
if not materials:
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
# 追加模式下,pkg.keywords 体现到目前为止累积的检索词(旧 + 新追加),
# 让前端列表 / 历史回显能看到完整的搜索路径。
pkg_keywords = keywords
if is_append_only and existing_pkg:
old_kws = list(existing_pkg.get("keywords") or [])
pkg_keywords = list(dict.fromkeys(old_kws + keywords))
pkg = MaterialPackage(keywords=pkg_keywords, materials=materials, summary=summary)
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
notices: list[dict] = []
@@ -187,11 +281,18 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
raw="\n\n----------\n\n".join(raw_dumps[:3]),
))
# 进度事件文案:追加模式下让用户看到只搜了哪几个新词
step_start_msg = (
f"追加检索关键词:{', '.join(keywords[:3])}"
if is_append_only
else f"搜索关键词:{', '.join(keywords[:3])}"
)
return {
"material_package": pkg,
"status": "awaiting_material_confirm",
# 补充检索:直接回写作(沿用原大纲);常规检索:进入素材确认暂停点
"status": "writing_draft" if is_supplement else "awaiting_material_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
{"type": "step_start", "agent_name": "素材收集专家", "message": step_start_msg},
*notices,
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
],
@@ -1,18 +1,54 @@
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import logging
import re
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, DraftArticle
from ..prompts.writer_prompts import DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
from ..prompts.writer_prompts import (
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
)
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser, safe_get_stream_writer
logger = logging.getLogger(__name__)
# 章节正文里的 Markdown 标题行(允许前导空白)
_HEADING_PREFIX_RE = re.compile(r"^[ \t]*#{1,6}[ \t]+", re.MULTILINE)
def _normalize_section_headings(text: str) -> str:
"""把章节正文里所有 Markdown 标题行统一压成三级标题(###)。
作家被要求只用 ### 作为「要点」小标题;模型偶尔误用 #/##/#### 时在此纠正,
保证文档标题层级符合「文章 # → 章节 ## → 要点 ###」,
前端目录栏据此才能把要点正确识别为二级标题。
"""
return _HEADING_PREFIX_RE.sub("### ", text)
# 疑似「拒绝 / 元话语」的强特征短语:模型把对用户说的话当成正文输出时出现,
# 这些短语几乎不可能出现在正常文章里,用作安全网识别。
_REFUSAL_MARKERS = (
"我无法完成", "无法完成此任务", "无法完成该任务", "无法完成这个任务",
"无法完成本次", "无法写出符合", "无法撰写这篇", "无法基于现有素材",
"无法基于当前素材", "无法基于提供的素材",
)
def _looks_like_refusal(text: str) -> bool:
"""判断一段章节产出是否疑似「拒绝 / 元话语」而非真正的文章正文。
模型遇到素材与意图不匹配时,可能把「我无法完成此任务」之类对用户说的话
当成正文输出。这里用强特征短语做启发式识别(安全网,非强校验)。
"""
if not text or not text.strip():
return False
return any(marker in text for marker in _REFUSAL_MARKERS)
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
@@ -27,6 +63,35 @@ def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
return "\n".join(lines) if lines else "(无相关素材)"
async def _save_draft_to_repo(config: RunnableConfig, draft: DraftArticle) -> None:
"""把 current_draft 异步落到业务表(best-effort,写失败不影响流程)。
``user_id`` 走 ``resolve_user_id(AUTO)`` 读 contextvar(FastAPI 请求线程设置,
asyncio.create_task 继承)。intent_parser 在阶段 4 已用同样规则建行;这里 update
时也用 AUTO 才能命中 owner filter。
"""
cfg = config.get("configurable", {}) or {}
thread_id = cfg.get("thread_id")
if not thread_id:
return
from deerflow.persistence.ai_writing_sessions import get_default_store
from deerflow.runtime.user_context import AUTO
repo = get_default_store()
if repo is None:
return
try:
await repo.update(
thread_id,
user_id=AUTO,
status="draft_ready",
draft_title=draft.get("title") or "",
draft_markdown=draft.get("full_markdown") or "",
)
except Exception:
logger.exception("writer_draft: save_draft failed (non-fatal) thread_id=%s", thread_id)
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
outline = state.get("current_outline")
if not outline:
@@ -39,10 +104,14 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
intervention = state.get("last_user_intervention") or {}
revision_count = state.get("revision_count", 0)
# SSE 流式推送所需
# 写作模式;用户在「素材不足求助」里选「按通用知识续写」的章节强制走宽松
writing_mode = state.get("writing_mode", "loose")
loose_titles: set[str] = set(intervention.get("_loose_sections") or [])
# 流式推送:阶段 2 起统一走 LangGraph 原生 custom 通道(get_stream_writer)。
# cfg 仍保留以便阶段 4 取 user_id / thread_id 用。
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
writer = safe_get_stream_writer()
# 修改意见(来自用户或编辑)
revision_notes = ""
@@ -89,6 +158,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
section_titles: list[str] = []
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
blocked_sections: list[dict] = [] # 严格模式下素材不足、无法写作的章节
for i, section in enumerate(sections):
sec_title = _section_title(i, section)
@@ -104,6 +174,10 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
for j, s in enumerate(sections)
)
# 本章节是否走严格模式(全局严格 且 未被用户标记为「通用知识续写」)
section_is_strict = writing_mode == "strict" and sec_title not in loose_titles
mode_section = WRITER_MODE_STRICT if section_is_strict else WRITER_MODE_LOOSE
user_msg = DRAFT_SECTION_USER.format(
article_title=article_title,
outline_context=outline_context,
@@ -111,6 +185,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
key_points=key_points_str,
section_materials=section_materials,
revision_section=revision_notes,
mode_section=mode_section,
target_words=words_per_section,
)
@@ -127,28 +202,45 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
for is_thinking, text in parser.feed(delta):
if not is_thinking:
section_buf += text
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": text,
"is_thinking": is_thinking,
"is_section_done": False,
}))
writer({
"type": "draft_chunk",
"section_index": i,
"section_title": sec_title,
"text": text,
"is_thinking": is_thinking,
"is_section_done": False,
})
# 章节完成
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": "",
"is_section_done": True,
}))
writer({
"type": "draft_chunk",
"section_index": i,
"section_title": sec_title,
"text": "",
"is_section_done": True,
})
cleaned_buf = section_buf.strip()
blocked_reason = ""
# 严格模式:作家用哨兵声明素材不足,或仍输出了拒绝话术 → 标记为待求助章节
if section_is_strict and SECTION_BLOCKED_MARKER in cleaned_buf:
blocked_reason = (
cleaned_buf.split(SECTION_BLOCKED_MARKER, 1)[1].strip()[:120]
or "素材不足以支撑本章节写作"
)
elif _looks_like_refusal(cleaned_buf):
if section_is_strict:
blocked_reason = "素材不足以支撑本章节写作"
else:
# 宽松模式:模型仍拒绝,安全网占位,不打断流程
logger.warning("draft section [%s] looks like a refusal, replaced with placeholder", sec_title)
section_errors.append(f"「{sec_title}」生成内容疑似拒绝/无关,已占位")
cleaned_buf = "(本节内容生成异常,建议重新生成或补充相关素材)"
if blocked_reason:
blocked_sections.append({"section_title": sec_title, "reason": blocked_reason})
cleaned_buf = f"(本节素材不足:{blocked_reason})"
section_titles.append(sec_title)
section_bufs.append(section_buf.strip())
section_bufs.append(cleaned_buf)
except Exception as e:
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
@@ -156,8 +248,26 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
section_bufs.append("(内容生成失败,请重试)")
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
# 严格模式下有章节无法基于素材写作 → 暂停求助用户,不产出半成品草稿
if blocked_sections:
return {
"blocked_sections": blocked_sections,
"status": "awaiting_section_help",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
{
"type": "notice", "agent_name": "作家", "title": "素材不足",
"detail": f"有 {len(blocked_sections)} 个章节缺少足够素材支撑写作,需要你决定如何处理。",
},
],
}
def _assemble() -> str:
body = "\n\n".join(f"## {t}\n\n{b}" for t, b in zip(section_titles, section_bufs))
# 章节正文里的标题统一归一到 ###(要点小标题),章节标题用 ##,文章标题用 #
body = "\n\n".join(
f"## {t}\n\n{_normalize_section_headings(b)}"
for t, b in zip(section_titles, section_bufs)
)
return f"# {article_title}\n\n{body}"
full_markdown = _assemble()
@@ -205,6 +315,11 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
revision_count=revision_count,
)
# 阶段 2:节点内并存 DB 写入(路由层 _stream_graph_segment 同样也会写一遍,
# 阶段 7 删冗余时移除路由层那份)。new path (LangGraph Server) 下路由层
# 不再介入,节点写就是唯一来源。
await _save_draft_to_repo(config, draft)
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
notices: list[dict] = []
if section_errors:
@@ -1,15 +1,17 @@
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import json
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, Outline, OutlineSection
from ..prompts.writer_prompts import OUTLINE_SYSTEM, OUTLINE_USER
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
from ..prompts.writer_prompts import (
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
logger = logging.getLogger(__name__)
@@ -42,25 +44,42 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
feedback_section = ""
if intervention.get("action") == "re_outline" and intervention.get("_feedback"):
feedback_section = f"\n重新规划要求:{intervention['_feedback']}"
feedback = ""
if intervention.get("action") == "re_outline":
feedback = intervention.get("_feedback") or intervention.get("outline_feedback") or ""
elif intervention.get("outline_feedback"):
feedback_section = f"\n重新规划要求:{intervention['outline_feedback']}"
feedback = intervention.get("outline_feedback") or ""
user_prompt = OUTLINE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
materials_summary=_materials_summary(pkg),
feedback_section=feedback_section,
)
# 重新规划且已有大纲 → 走「编辑模式」:把当前大纲喂给模型,按用户要求调整,
# 而非从零重生成(否则「加一节/删一节」这类指令没有可编辑的基准)。
current_outline = state.get("current_outline")
is_revise = bool(feedback) and bool(current_outline)
# SSE 流式推送
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
if is_revise:
system_prompt = OUTLINE_REVISE_SYSTEM
user_prompt = OUTLINE_REVISE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
current_outline=json.dumps(current_outline, ensure_ascii=False, indent=2),
feedback=feedback,
materials_summary=_materials_summary(pkg),
)
else:
system_prompt = OUTLINE_SYSTEM
feedback_section = f"\n重新规划要求:{feedback}" if feedback else ""
user_prompt = OUTLINE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
materials_summary=_materials_summary(pkg),
feedback_section=feedback_section,
)
# 流式推送:阶段 2 起统一走 LangGraph 原生 custom 通道(get_stream_writer)。
writer = safe_get_stream_writer()
full_text = "" # 只累积正文(非思考)用于 JSON 解析
parser = ThinkTagParser()
@@ -70,7 +89,7 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
model = get_model(config)
async for chunk in model.astream([
SystemMessage(content=OUTLINE_SYSTEM),
SystemMessage(content=system_prompt),
HumanMessage(content=user_prompt),
]):
delta = _content_to_text(chunk.content)
@@ -79,12 +98,11 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("outline_chunk", {
"session_id": session_id,
"text": text,
"is_thinking": is_thinking,
}))
writer({
"type": "outline_chunk",
"text": text,
"is_thinking": is_thinking,
})
except Exception as e:
logger.error("outline streaming failed: %s", e, exc_info=True)
return {
@@ -98,14 +116,18 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
}],
}
fallback = {
"title": "未命名文章",
"sections": [
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
],
}
# 编辑模式解析失败时回退为「当前大纲」,避免把用户已有大纲弄丢
if is_revise:
fallback = current_outline
else:
fallback = {
"title": "未命名文章",
"sections": [
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
],
}
result = parse_json(full_text, fallback)
outline = _parse_outline(result)
@@ -1,9 +1,13 @@
# 注:两个 USER 提示词末尾的 `/no_think` 是 Qwen3 系列的"软开关",
# 收到当轮即跳过思考过程;对其它大模型只是一行普通文本、无副作用。
# 中文「严禁思考」强令块对所有模型生效,保留不动。
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:为写作任务提取搜索关键词。
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
你只能做以下事情:
1. 从写作意图中提取 3-6 个搜索关键词
1. 从写作意图中按要求的数量提取搜索关键词
你不能写文章,不能评价内容好坏,只做关键词提取。
【严禁思考】
@@ -25,8 +29,13 @@ EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
请提取 3-6 个搜索关键词,用于搜集相关素材。只输出 JSON:
{{"keywords": ["关键词1", "关键词2"]}}"""
请提取 {keyword_count} 个搜索关键词,用于搜集相关素材。
关键词数量必须为 {keyword_count} 个,不多不少。
关键词必须是依据上述写作意图提炼的真实检索词,不要照抄下方 JSON 模板里的占位符。
只输出 JSON(下方 ... 是占位示意,不是真实关键词):
{{"keywords": ["...", "..."]}}
/no_think"""
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:对搜索结果按相关度排序和筛选。
@@ -56,4 +65,6 @@ RANK_MATERIALS_USER = """写作意图:{user_intent}
{materials_list}
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}"""
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}
/no_think"""
@@ -5,7 +5,17 @@ OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
你只能做以下事情:
1. 分析素材,提取核心观点
2. 规划文章章节结构和每节要点
你不能写正文,只写大纲。
你不能写正文,只写大纲。大纲须紧扣写作意图,并尽量覆盖可用素材中的核心信息。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
- 禁止输出任何对用户说的话或对任务的评价(如"素材不足""无法规划大纲");无论素材是否充分,都必须直接产出大纲 JSON。
- 收到输入后立即、直接输出最终 JSON,不得有任何前置内容。
【要点(key_points)的特殊要求】
- 每个要点都会作为文章里的一个二级小标题呈现,所以要点必须写成"标题式短语"。
- 用简洁的名词性短语,控制在 15 字以内,不要写成完整句子,不要带句末标点。
- 每个章节规划 2-4 个要点,不要过多,避免文章被切得过碎。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
@@ -17,9 +27,9 @@ OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
{
"title": "新能源汽车出口为何高速增长",
"sections": [
{"section_title": "引言:出口数据创新高", "key_points": ["今年出口量同比增长", "在全球市场的排名变化"], "material_ids": ["m001"]},
{"section_title": "引言:出口数据创新高", "key_points": ["出口量同比增幅", "全球市场排名变化"], "material_ids": ["m001"]},
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
{"section_title": "结论与展望", "key_points": ["未来增长面临的挑战", "行业趋势判断"], "material_ids": ["m004"]}
{"section_title": "结论与展望", "key_points": ["未来增长挑战", "行业趋势判断"], "material_ids": ["m004"]}
]
}"""
@@ -33,21 +43,98 @@ OUTLINE_USER = """写作意图:{user_intent}
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
# ── 重新规划(基于现有大纲的编辑)─────────────────────────────────────────────
OUTLINE_REVISE_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据用户的修改要求,调整一份【已有的文章大纲】。
你的工作背景:你有十年内容创作经验,擅长精准理解修改意图并落实到大纲结构上。
你只能做以下事情:
1. 在「当前大纲」基础上,按「用户修改要求」做调整
你不能写正文,只输出调整后的大纲。
【修改原则】(最高优先级)
- 准确识别并落实用户意图,常见操作:新增章节、删除章节、合并章节、拆分章节、调整章节顺序、修改章节标题、调整章节要点。
- 只改用户明确提到的部分:用户没要求改动的章节,必须原样保留(标题、要点、material_ids 全部不变)。
- 新增章节时:紧扣写作意图补充合理章节,并尽量关联可用素材的 material_ids。
- 章节数量完全由用户要求决定,没有上下限——用户要加就加、要减就减。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
- 禁止输出任何对用户说的话;收到输入后立即、直接输出最终 JSON。
【要点(key_points)的特殊要求】
- 每个要点会作为文章里的二级小标题呈现,写成简洁名词性短语,15 字以内,不带句末标点。
- 每个章节 2-4 个要点。
【输出要求】
- 只输出一个调整后的【完整】大纲 JSON 对象,不要解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
- 结构与「当前大纲」一致:
{"title": "...", "sections": [{"section_title": "...", "key_points": ["..."], "material_ids": ["..."]}]}"""
OUTLINE_REVISE_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
目标字数:{word_count_target}字
当前大纲(JSON):
{current_outline}
用户修改要求:{feedback}
可用素材摘要:
{materials_summary}
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据大纲和素材,写出一段高质量的文章内容。
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
你只能做以下事情:
1. 根据给定的章节要点和素材写这一段内容
2. 观点必须来自素材,不编造数据
2. 优先采用素材中的事实与数据;素材不足时可结合你的通用知识与常识补充论述,但须紧扣写作意图与章节主题,不要脱离主题杜撰具体的数字、人名、机构名与日期
你不能修改大纲,不能写其他章节,只写当前这一段。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止"让我想想""首先""我需要""根据写作要求"之类的过渡语。
- 禁止输出 <think></think> 标签及其中的任何内容。
- 禁止输出任何对用户说的话或对任务本身的评价,例如"我无法完成此任务""您提供的素材无法支撑写作""建议您提供……素材""素材与意图不匹配"等——这类话绝不允许出现在文章里。
- 无论素材是否充分,你都必须直接产出文章正文本身;收到输入后立即开始写正文(含 ### 小标题),不要有任何前置说明。
【结构要求】
- 把本章节的每一个「章节要点」写成一个二级小标题,用 Markdown 三级标题语法:### 要点
- 每个 ### 小标题下面紧跟该要点的论述正文,正文连贯成段。
- 即使章节只有一个要点,也要为它写出对应的 ### 小标题。
- 小标题文字简洁,直接沿用给定的要点措辞,不要加序号、不要加标点。
- 只能使用 ### 这一级标题;绝不要输出 # 或 ##(文章标题、章节标题由系统统一添加)。
【输出要求】
- 直接返回这一段的正文文字,不要 JSON,不要章节标题,不要解释。
- 直接返回这一段的内容(含 ### 小标题与正文),不要 JSON,不要解释。
- 不要写"以下是正文""本段内容如下"之类的开场白。
- 段落要连贯成文,不要罗列要点。
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
- 充分展开每个要点:每个要点至少用 2-3 句话论述,结合素材中的事实与数据说理,写满要求的篇幅。"""
- 充分展开每个要点:每个要点至少用 2-3 句话论述,尽量结合素材中的事实与数据说理,写满要求的篇幅。"""
# ── 写作模式说明:注入 DRAFT_SECTION_USER 的 {mode_section} ───────────────────
# 章节无法基于素材写作时,作家在严格模式下输出的哨兵标记
SECTION_BLOCKED_MARKER = "[[SECTION_BLOCKED]]"
WRITER_MODE_LOOSE = """
【写作模式:宽松】
素材充分时优先采用素材;素材不足时可结合你的通用知识与常识补充论述,
但必须紧扣写作意图与本章节主题,不要脱离主题杜撰具体数字、人名、机构名。"""
WRITER_MODE_STRICT = """
【写作模式:严格依据素材】
本章节论述必须严格基于上方「相关素材」,不得使用素材之外的知识,不得杜撰任何事实与数据。
如果上方素材确实不足以支撑本章节的写作,不要勉强拼凑、不要写无关内容,
此时只输出一行:{marker} 一句话说明缺什么素材(例:{marker} 缺少关于XX的具体数据与案例),
此外不要输出任何内容。只要素材能支撑哪怕部分要点,就正常写作,不要输出该标记。""".format(
marker=SECTION_BLOCKED_MARKER
)
DRAFT_SECTION_USER = """文章标题:{article_title}
文章整体结构:
@@ -58,8 +145,9 @@ DRAFT_SECTION_USER = """文章标题:{article_title}
相关素材:
{section_materials}
{revision_section}
请写出「{section_title}」这一章节的正文内容,不少于 {target_words} 字,论述充分、内容充实。"""
{revision_section}{mode_section}
请写出「{section_title}」这一章节的内容,不少于 {target_words} 字。
把上述每个章节要点写成一个 ### 二级小标题,并在标题下展开论述,论述充分、内容充实。"""
DRAFT_EXPAND_USER = """文章标题:{article_title}
当前章节:{section_title}
@@ -71,5 +159,8 @@ DRAFT_EXPAND_USER = """文章标题:{article_title}
这一章节已有内容如下:
{existing_text}
以上内容篇幅不足。请在保留原意和事实的基础上重写并扩充这一章节,使正文不少于 {target_words} 字。
要求:补充更多来自素材的事实、数据与分析,论述更充分、更具体;直接返回扩充后的完整章节正文,不要章节标题、不要解释。"""
以上内容篇幅不足。请在保留原意、事实和 ### 二级小标题结构的基础上重写并扩充这一章节,使内容不少于 {target_words} 字。
要求:
- 补充更多来自素材的事实、数据与分析,论述更充分、更具体。
- 保留并沿用原有的 ### 小标题(每个章节要点对应一个),只用 ### 这一级标题,不要输出 # 或 ##。
- 直接返回扩充后的完整章节内容(含 ### 小标题),不要解释。"""
@@ -73,17 +73,27 @@ class UserInterventionPayload(TypedDict):
WritingStatus = Literal[
"researching", "awaiting_material_confirm",
"writing_outline", "awaiting_outline_confirm",
"writing_draft", "awaiting_draft_confirm",
"writing_draft", "awaiting_section_help", "awaiting_draft_confirm",
"reviewing", "awaiting_review_confirm",
"revising", "done", "error",
]
class BlockedSection(TypedDict):
"""严格模式下素材不足、无法写作的章节。"""
section_title: str
reason: str
class AIWritingState(TypedDict):
# 写作配置
user_intent: str
article_type: str
target_audience: str
word_count_target: int
# 素材检索关键词数量(用户指定,1~10,默认 6)
keyword_count: int
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
writing_mode: str
# Agent A 输出
material_package: Optional[MaterialPackage]
@@ -99,6 +109,9 @@ class AIWritingState(TypedDict):
# 用户干预数据(每个干预点会覆盖)
last_user_intervention: Optional[UserInterventionPayload]
# 严格模式下素材不足、待用户决策的章节(写作节点产出,求助暂停点消费)
blocked_sections: Optional[List[BlockedSection]]
# 流程控制
revision_count: int
max_revisions: int
@@ -268,6 +268,7 @@ def _build_middlewares(
# Add TodoList middleware if plan mode is enabled
cfg = _get_runtime_config(config)
is_plan_mode = cfg.get("is_plan_mode", False)
is_bootstrap = cfg.get("is_bootstrap", False)
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
if todo_list_middleware is not None:
middlewares.append(todo_list_middleware)
@@ -279,8 +280,10 @@ def _build_middlewares(
# Add TitleMiddleware
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
# Add MemoryMiddleware (after TitleMiddleware)
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory))
# Add MemoryMiddleware (after TitleMiddleware) — skip in bootstrap mode (agent creation flow)
memory_injection_enabled = cfg.get("memory_injection_enabled", True)
if not is_bootstrap:
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
# Add ViewImageMiddleware only if the current model supports vision.
# Use the resolved runtime model_name from make_lead_agent to avoid stale config values.
@@ -309,6 +312,37 @@ def _build_middlewares(
middlewares.append(SetupAgentApprovalMiddleware())
# SkillStopMiddleware — intercepts skill invocations and halts execution
# when a configured skill is about to run. Request-level options
# (skill_stop_names / skill_stop_message) take precedence; static
# config in ``app_config.skill_stop`` is the fallback.
skill_stop_names = cfg.get("skill_stop_names")
skill_stop_message = cfg.get("skill_stop_message")
if skill_stop_names:
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
middlewares.append(
SkillStopMiddleware(
skill_names=skill_stop_names,
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
stop_message=skill_stop_message,
stop_on_match=True,
)
)
logger.info("SkillStopMiddleware enabled (from request): %s", skill_stop_names)
elif resolved_app_config.skill_stop.enabled and resolved_app_config.skill_stop.skill_names:
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
middlewares.append(
SkillStopMiddleware(
skill_names=resolved_app_config.skill_stop.skill_names,
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
stop_message=resolved_app_config.skill_stop.stop_message,
stop_on_match=resolved_app_config.skill_stop.stop_on_match,
)
)
logger.info("SkillStopMiddleware enabled (from config): %s", resolved_app_config.skill_stop.skill_names)
# ClarificationMiddleware should always be last
middlewares.append(ClarificationMiddleware())
return middlewares
@@ -336,13 +370,19 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
subagent_enabled = cfg.get("subagent_enabled", False)
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
writing_mode = bool(cfg.get("writing_mode", False))
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
writing_artifact_path = None
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
requested_excluded_tools = cfg.get("excluded_tools") or None
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
try:
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
except FileNotFoundError:
logger.warning("Agent directory not found for '%s'; running with default config", agent_id)
agent_config = None
agent_display_name = agent_config.name if agent_config else agent_id
# Custom agent model from agent config (if any), or None to let _resolve_model_name pick the default
agent_model_name = agent_config.model if agent_config and agent_config.model else None
@@ -375,7 +415,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
config["metadata"] = {}
if is_bootstrap:
runtime_available_skills: list[str] | None = ["bootstrap"]
runtime_available_skills: list[str] | None = None # all enabled skills visible in bootstrap
elif agent_config:
# Custom agents must use an explicit allowlist. If omitted, default to none.
runtime_available_skills = list(agent_config.skills or [])
@@ -407,6 +447,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
model_name=model_name,
subagent_enabled=subagent_enabled,
is_scheduled_run=cfg.get("is_scheduled_run", False),
excluded_tools=requested_excluded_tools,
app_config=resolved_app_config,
)
+ [setup_agent],
@@ -416,8 +457,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
max_concurrent_subagents=max_concurrent_subagents,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
available_skills=set(["bootstrap"]),
available_skills=None,
is_bootstrap=True,
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
),
state_schema=ThreadState,
)
@@ -430,6 +473,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
groups=agent_config.tool_groups if agent_config else None,
subagent_enabled=subagent_enabled,
is_scheduled_run=cfg.get("is_scheduled_run", False),
excluded_tools=requested_excluded_tools,
app_config=resolved_app_config,
),
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
@@ -442,6 +486,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
writing_artifact_path=writing_artifact_path,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
),
state_schema=ThreadState,
)
@@ -464,11 +464,22 @@ You are {agent_name}, an internal super agent.
- ❌ DO NOT skip clarification for "efficiency" - accuracy matters more than speed
- ❌ DO NOT make assumptions when information is missing - ALWAYS ask
- ❌ DO NOT proceed with guesses - STOP and call ask_clarification first
- ❌ DO NOT cram several unrelated questions into one `question` string — split them into separate tool calls
- ✅ Analyze the request in thinking → Identify unclear aspects → Ask BEFORE any action
- ✅ If you identify the need for clarification in your thinking, you MUST call the tool IMMEDIATELY
- ✅ After calling ask_clarification, execution will be interrupted automatically
- ✅ Wait for user response - do NOT continue with assumptions
**Batching multiple independent questions:**
- When the request leaves **several independent aspects** unclear (e.g. project type AND tech stack
AND deployment target), call `ask_clarification` multiple times in the same turn — one call per
aspect, each with its own focused `question` and `options`.
- The frontend groups all clarification cards from one turn into a single submission, so the user
answers them together in one reply. You will then receive ONE combined user message addressing
every question in order — read it carefully and match answers back to the questions you asked.
- If the next question only makes sense after the previous answer (sequential dependency), ask
only the first one this turn and wait for the reply before asking the next.
**How to Use:**
```python
ask_clarification(
@@ -496,6 +507,8 @@ You: "Deploying to staging..." [proceed]
{skills_section}
{bootstrap_section}
{deferred_tools_section}
{writing_mode_section}
@@ -602,7 +615,7 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
"""
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
"""组装注入系统提示词的记忆上下文 (Memory V2)。
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
@@ -612,10 +625,13 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
Args:
agent_name: agent 标识;``None`` 时按 "default" 处理。
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
injection_enabled: 运行时开关;``False`` 时跳过注入直接返回空串。
Returns:
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
"""
if not injection_enabled:
return ""
try:
from deerflow.agents.memory.manager import build_memory_manager
from deerflow.config.memory_config import get_effective_memory_config
@@ -819,6 +835,27 @@ def _build_custom_mounts_section(*, app_config: AppConfig | None = None) -> str:
return f"\n**Custom Mounted Directories:**\n{mounts_list}\n- If the user needs files outside `/mnt/user-data`, use these absolute container paths directly when they match the requested directory"
def _build_bootstrap_section() -> str:
return """<bootstrap_agent_creation>
你正在帮助用户创建一个自定义智能体。
**技能匹配规则(必须遵守):**
1. 先调用 `skill_list` 查看用户当前所有可用技能及其描述。
2. 根据用户对新智能体的描述/意图,从已有技能中选出最相关的 1-5 个技能名称(skill name,使用 hyphen-case)。
3. 将选出的技能名称填入 `setup_agent` 的 `skills` 参数。
4. **严禁在本次创建流程中调用 `skill_manage`(创建、编辑或删除技能)**。如用户提出需要新技能,请告知他们创建完智能体后再单独创建。
5. 如果没有任何技能与用户意图相关,可将 `skills` 设为空列表,并向用户说明原因。
**工作流程:**
1. 与用户充分沟通,明确新智能体的用途、专长和边界。
2. 调用 `skill_list` 列出现有技能。
3. 在思考中匹配技能,选出最合适的。
4. 草拟 SOUL.md(定义智能体的人设、工作方式、输出要求)。
5. 向用户确认配置后,调用 `setup_agent` 提交。
</bootstrap_agent_creation>
"""
def apply_prompt_template(
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
@@ -828,12 +865,14 @@ def apply_prompt_template(
writing_mode: bool = False,
writing_artifact_path: str | None = None,
available_skills: set[str] | None = None,
is_bootstrap: bool = False,
app_config: AppConfig | None = None,
memory_injection_enabled: bool = True,
) -> str:
storage_agent_id = agent_id if agent_id is not None else agent_name
# Get memory context
memory_context = _get_memory_context(storage_agent_id, app_config=app_config)
memory_context = _get_memory_context(storage_agent_id, app_config=app_config, injection_enabled=memory_injection_enabled)
# Include subagent section only if enabled (from runtime parameter)
n = max_concurrent_subagents
@@ -857,6 +896,9 @@ def apply_prompt_template(
else ""
)
# Bootstrap section: skill-matching guidance (only in agent creation flow)
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
# Get skills section
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
@@ -902,6 +944,7 @@ Writing mode is enabled for this run.
agent_name=agent_name or storage_agent_id or "CM 助手",
soul=get_agent_soul(storage_agent_id),
skills_section=skills_section,
bootstrap_section=bootstrap_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
@@ -223,8 +223,11 @@ def build_memory_manager(
) -> MemoryManager:
"""按生效配置组装并初始化一个 MemoryManager。
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
这是构建 manager 的统一入口 —— 根据 ``config.version`` 选择 V1 或 V2 provider:
- ``version="v1"``: 仅注册 V1BuiltinProvider (memory.json + LLM 自动提取)
- ``version="v2"``: 注册 BuiltinFileProvider + 可选 HindsightProvider (USER.md/MEMORY.md)
所有 provider 已 ``initialize`` 完毕,可直接使用。
Parameters
@@ -234,7 +237,6 @@ def build_memory_manager(
config:
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.config.paths import get_paths
@@ -242,34 +244,53 @@ def build_memory_manager(
config = get_effective_memory_config(user_id)
manager = MemoryManager()
base_dir = str(get_paths().base_dir)
if config.builtin.enabled:
if getattr(config, "version", "v2") == "v1":
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
v1_cfg = config.v1
manager.add_provider(
BuiltinFileProvider(
memory_char_limit=config.builtin.memory_char_limit,
user_char_limit=config.builtin.user_char_limit,
deduplicate_on_load=config.builtin.deduplicate_on_load,
security=config.security,
V1BuiltinProvider(
max_facts=v1_cfg.max_facts,
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
debounce_seconds=v1_cfg.debounce_seconds,
model_name=v1_cfg.model_name,
token_budget=v1_cfg.token_budget,
)
)
else:
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
if config.provider == "hindsight":
try:
from deerflow.agents.memory.providers.hindsight import HindsightProvider
if config.builtin.enabled:
manager.add_provider(
BuiltinFileProvider(
memory_char_limit=config.builtin.memory_char_limit,
user_char_limit=config.builtin.user_char_limit,
deduplicate_on_load=config.builtin.deduplicate_on_load,
security=config.security,
)
)
hindsight = HindsightProvider(config.hindsight)
if hindsight.is_available():
manager.add_provider(hindsight)
else:
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
if config.provider == "hindsight":
try:
from deerflow.agents.memory.providers.hindsight import HindsightProvider
hindsight = HindsightProvider(config.hindsight)
if hindsight.is_available():
manager.add_provider(hindsight)
else:
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
manager.initialize_all(
user_id=user_id,
agent_id=agent_id or "default",
agent_id=agent_id,
thread_id=thread_id,
base_dir=str(get_paths().base_dir),
base_dir=base_dir,
)
return manager
@@ -69,3 +69,42 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
filtered.append(msg)
return filtered
_CORRECTION_PATTERNS = re.compile(
r"\b(?:no[,.]?\s+(?:that'?s?\s+)?(?:wrong|incorrect|not\s+right|not\s+what)|"
r"actually[,.]?\s+(?:it\s+(?:is|should)|you\s+(?:should|need|must)|the\s+correct)|"
r"that'?s?\s+(?:wrong|incorrect|not\s+right|not\s+what\s+i)|"
r"you\s+(?:got\s+it\s+wrong|misunderstood|made\s+an?\s+error|made\s+a\s+mistake)|"
r"incorrect[,.]|wrong[,.]|mistake[,.]|correction:|that\s+is\s+not\s+right)\b",
re.IGNORECASE,
)
_REINFORCEMENT_PATTERNS = re.compile(
r"\b(?:perfect[,.]?|exactly[,.]?|correct[,.]?|that'?s?\s+(?:right|correct|perfect|exactly\s+right)|"
r"you\s+(?:got\s+it|nailed\s+it)|great\s+(?:job|work)|well\s+done|"
r"yes[,.]?\s+(?:that'?s?\s+(?:right|correct|exactly)|exactly)|"
r"keep\s+(?:it\s+up|doing\s+that|going)|"
r"(?:that'?s?\s+|this\s+is\s+)?(?:exactly|precisely)\s+(?:what\s+i|right))\b",
re.IGNORECASE,
)
def detect_correction(messages: list[Any]) -> bool:
"""Detect if the conversation contains explicit user correction signals."""
for msg in messages:
if getattr(msg, "type", None) == "human":
text = extract_message_text(msg)
if _CORRECTION_PATTERNS.search(text):
return True
return False
def detect_reinforcement(messages: list[Any]) -> bool:
"""Detect if the conversation contains positive reinforcement signals."""
for msg in messages:
if getattr(msg, "type", None) == "human":
text = extract_message_text(msg)
if _REINFORCEMENT_PATTERNS.search(text):
return True
return False
@@ -0,0 +1,161 @@
"""Memory V1 — 原版 DeerFlow JSON-based MemoryProvider 实现。
存储格式:``{base_dir}/users/{user_id}/memory.json``(经 FileMemoryStorage 管理)
JSON 结构使用嵌套 ``{summary, updatedAt}`` 对象,与原版 DeerFlow 完全一致。
LLM 在 V1 模式下不调用 memory 工具 —— 事实完全由异步 LLM 提取。
"""
from __future__ import annotations
import logging
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
logger = logging.getLogger(__name__)
class _FakeMsg:
"""轻量 message 对象,用于将文本传给原版 format_conversation_for_update。"""
__slots__ = ("type", "content")
def __init__(self, type_: str, content: str) -> None:
self.type = type_
self.content = content
class V1BuiltinProvider(MemoryProvider):
"""原版 DeerFlow V1 记忆 provider。"""
def __init__(
self,
*,
max_facts: int = 100,
fact_confidence_threshold: float = 0.7,
debounce_seconds: int = 30,
model_name: str = "",
token_budget: int = 2000,
) -> None:
self._max_facts = max_facts
self._confidence_threshold = fact_confidence_threshold
self._debounce_seconds = debounce_seconds
self._model_name = model_name
self._token_budget = token_budget
self._user_id: str = ""
self._thread_id: str = ""
self._agent_id: str | None = None
self._frozen_block: str = ""
@property
def name(self) -> str:
return "v1_builtin"
def is_available(self) -> bool:
return True
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str,
**kwargs: Any,
) -> None:
_ = base_dir # FileMemoryStorage uses get_paths() internally
self._user_id = user_id
self._thread_id = thread_id or ""
self._agent_id = agent_id
self._frozen_block = self._build_frozen_block()
# ---- 系统提示词 -------------------------------------------------------
def _build_frozen_block(self) -> str:
"""从 memory.json 读取数据,用 format_memory_for_injection 渲染注入块。
当 agent_id 已知时加载 per-agent 记忆;否则加载用户级记忆。
"""
try:
from deerflow.agents.memory.v1.prompt import format_memory_for_injection
from deerflow.agents.memory.v1.storage import get_memory_storage
data = get_memory_storage().load(
agent_name=self._agent_id or None,
user_id=self._user_id,
)
text = format_memory_for_injection(data, max_tokens=self._token_budget)
if not text:
return ""
return f"<memory>\n{text}\n</memory>"
except Exception as exc: # noqa: BLE001
logger.warning("V1 provider 加载 memory.json 失败: %s", exc)
return ""
def system_prompt_block(self) -> str:
return self._frozen_block
# ---- 召回 / 写入 ------------------------------------------------------
def prefetch(self, query: str, *, thread_id: str = "") -> str:
# V1 无动态召回,静态 snapshot 已在 system_prompt_block 里
return ""
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
turn_index: int | None = None,
) -> None:
"""把本轮对话加入防抖队列,等待 LLM 异步提取事实。"""
if not user_content.strip() and not assistant_content.strip():
return
try:
from deerflow.agents.memory.v1.queue import get_memory_queue
msgs = [
_FakeMsg("human", user_content),
_FakeMsg("ai", assistant_content),
]
queue = get_memory_queue()
queue.add(
thread_id=thread_id or self._thread_id,
messages=msgs,
user_id=self._user_id,
agent_name=self._agent_id or None,
)
except Exception as exc: # noqa: BLE001
logger.warning("V1 provider sync_turn 入队失败: %s", exc)
# ---- 工具 (V1 不暴露工具给 LLM) -------------------------------------
def get_tool_schemas(self) -> list[dict[str, Any]]:
return []
# ---- API 读取接口 (供 Gateway router 用) -----------------------------
def read_v1_data(self) -> dict[str, Any]:
"""读取 memory.json 的当前内容(实时读磁盘,非 frozen snapshot)。
当 agent_id 已知时读取 per-agent 记忆;否则读用户级记忆。
"""
from deerflow.agents.memory.v1.storage import get_memory_storage
return get_memory_storage().load(
agent_name=self._agent_id or None,
user_id=self._user_id,
)
def fact_count(self) -> int:
try:
return len(self.read_v1_data().get("facts", []))
except Exception: # noqa: BLE001
return 0
def usage_text(self) -> str:
return f"{self.fact_count()}/{self._max_facts} 个事实"
__all__ = ["V1BuiltinProvider"]
@@ -0,0 +1 @@
"""Memory V1 — 原版 DeerFlow JSON-based 记忆子模块。"""
@@ -0,0 +1,371 @@
"""Prompt templates for memory update and injection."""
import math
import re
from typing import Any
try:
import tiktoken
TIKTOKEN_AVAILABLE = True
except ImportError:
TIKTOKEN_AVAILABLE = False
# Prompt template for updating memory based on conversation
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
Current Memory State:
<current_memory>
{current_memory}
</current_memory>
New Conversation to Process:
<conversation>
{conversation}
</conversation>
Instructions:
1. Analyze the conversation for important information about the user
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
3. Update the memory sections as needed following the detailed length guidelines below
Before extracting facts, perform a structured reflection on the conversation:
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
If yes, record them as facts with the most appropriate category and confidence.
{correction_hint}
Memory Section Guidelines:
**User Context** (Current state - concise summaries):
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
Example: Core contributor, project names with metrics (16k+ stars), technical stack
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
Example: Bilingual capabilities, specific interest areas, expertise domains
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
Include: Active implementation work, troubleshooting issues, market/research interests
Note: This captures SEVERAL concurrent focus areas, not just one task
**History** (Temporal context - rich paragraphs; updated incrementally each conversation):
- recentMonths: Running log of recent work and activities (4-6 sentences or 1-2 paragraphs)
Scope: Accumulate from each conversation — any task the user worked on, problems solved, technologies touched
Update policy: Set shouldUpdate=true whenever the conversation contains any task, discussion, or work activity; integrate new content into existing text rather than overwriting
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
- earlierContext: Important patterns promoted from recentMonths over time (3-5 sentences or 1 paragraph)
Scope: Recurring themes and established patterns across multiple conversations
Update policy: Set shouldUpdate=true when a pattern repeats or an older item in recentMonths should be archived here
Include: Past projects, learning journeys, established patterns
- longTermBackground: Persistent foundational context that rarely changes (2-4 sentences)
Scope: Core expertise, fundamental working style, long-standing interests
Update policy: Set shouldUpdate=true only when a genuinely new foundational fact emerges
Include: Core expertise, longstanding interests, fundamental working style
**Facts Extraction**:
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
- Include proper nouns (company names, project names, technology names)
- Preserve technical terminology and version numbers
- Categories:
* preference: Tools, styles, approaches user prefers/dislikes
* knowledge: Specific expertise, technologies mastered, domain knowledge
* context: Background facts (job title, projects, locations, languages)
* behavior: Working patterns, communication habits, problem-solving approaches
* goal: Stated objectives, learning targets, project ambitions
* correction: Explicit agent mistakes or user corrections, including the correct approach
- Confidence levels:
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
* 0.7-0.8: Strongly implied from actions/discussions
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
**What Goes Where**:
- workContext: Current job, active projects, primary tech stack
- personalContext: Languages, personality, interests outside direct work tasks
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
- recentMonths: Cumulative log of recent work — update from EVERY conversation with meaningful content
- earlierContext: Recurring patterns promoted from recentMonths; update when patterns solidify
- longTermBackground: Unchanging foundational facts about the user; update only for new foundational info
**Language Requirement**:
- ALL summaries and fact content MUST be written in Chinese (Simplified)
- Keep technical terms, proper nouns, product names, and company names in their original form (e.g., DeepSeek, LangGraph, GitHub)
- Surrounding descriptive text must be in Chinese regardless of the conversation language
Output Format (JSON):
{{
"user": {{
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
}},
"history": {{
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
}},
"newFacts": [
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
],
"factsToRemove": ["fact_id_1", "fact_id_2"]
}}
Important Rules:
- For recentMonths: set shouldUpdate=true whenever the conversation contains any task, work, or substantive discussion — this field accumulates like a log
- For earlierContext / longTermBackground: set shouldUpdate=true only when a new pattern solidifies or a foundational fact changes
- For user context fields: set shouldUpdate=true if there's meaningful new information
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
- Include specific metrics, version numbers, and proper nouns in facts
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
- Remove facts that are contradicted by new information
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
Keep 3-5 concurrent focus themes that are still active and relevant
- For history sections, integrate new information chronologically into appropriate time period
- Preserve technical accuracy - keep exact names of technologies, companies, projects
- Focus on information useful for future interactions and personalization
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
session-specific and ephemeral — they will not be accessible in future sessions.
Recording upload events causes confusion in subsequent conversations.
- CRITICAL: Write ALL summary text and fact content in Chinese (Simplified). Technical terms and proper nouns (product names, company names, technology names) may remain in their original language, but all other text must be in Chinese.
Return ONLY valid JSON, no explanation or markdown."""
# Prompt template for extracting facts from a single message
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
Message:
{message}
Extract facts in this JSON format:
{{
"facts": [
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
]
}}
Categories:
- preference: User preferences (likes/dislikes, styles, tools)
- knowledge: User's expertise or knowledge areas
- context: Background context (location, job, projects)
- behavior: Behavioral patterns
- goal: User's goals or objectives
- correction: Explicit corrections or mistakes to avoid repeating
Rules:
- Only extract clear, specific facts
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
- Skip vague or temporary information
- Write all fact content in Chinese (Simplified); technical terms and proper nouns may remain in their original language
Return ONLY valid JSON."""
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
"""Count tokens in text using tiktoken.
Args:
text: The text to count tokens for.
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
Returns:
The number of tokens in the text.
"""
if not TIKTOKEN_AVAILABLE:
return len(text) // 4
try:
encoding = tiktoken.get_encoding(encoding_name)
return len(encoding.encode(text))
except Exception:
return len(text) // 4
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
"""Coerce a confidence-like value to a bounded float in [0, 1].
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
to the default before clamping, preventing them from dominating ranking.
The ``default`` parameter is assumed to be a finite value.
"""
try:
confidence = float(value)
except (TypeError, ValueError):
return max(0.0, min(1.0, default))
if not math.isfinite(confidence):
return max(0.0, min(1.0, default))
return max(0.0, min(1.0, confidence))
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
"""Format memory data for injection into system prompt.
Args:
memory_data: The memory data dictionary.
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
Returns:
Formatted memory string for system prompt injection.
"""
if not memory_data:
return ""
sections = []
# Format user context
user_data = memory_data.get("user", {})
if user_data:
user_sections = []
work_ctx = user_data.get("workContext", {})
if work_ctx.get("summary"):
user_sections.append(f"Work: {work_ctx['summary']}")
personal_ctx = user_data.get("personalContext", {})
if personal_ctx.get("summary"):
user_sections.append(f"Personal: {personal_ctx['summary']}")
top_of_mind = user_data.get("topOfMind", {})
if top_of_mind.get("summary"):
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
if user_sections:
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
# Format history
history_data = memory_data.get("history", {})
if history_data:
history_sections = []
recent = history_data.get("recentMonths", {})
if recent.get("summary"):
history_sections.append(f"Recent: {recent['summary']}")
earlier = history_data.get("earlierContext", {})
if earlier.get("summary"):
history_sections.append(f"Earlier: {earlier['summary']}")
background = history_data.get("longTermBackground", {})
if background.get("summary"):
history_sections.append(f"Background: {background['summary']}")
if history_sections:
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
# Format facts (sorted by confidence; include as many as token budget allows)
facts_data = memory_data.get("facts", [])
if isinstance(facts_data, list) and facts_data:
ranked_facts = sorted(
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
reverse=True,
)
base_text = "\n\n".join(sections)
base_tokens = _count_tokens(base_text) if base_text else 0
facts_header = "Facts:\n"
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
running_tokens = base_tokens + separator_tokens
fact_lines: list[str] = []
for fact in ranked_facts:
content_value = fact.get("content")
if not isinstance(content_value, str):
continue
content = content_value.strip()
if not content:
continue
category = str(fact.get("category", "context")).strip() or "context"
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
source_error = fact.get("sourceError")
if category == "correction" and isinstance(source_error, str) and source_error.strip():
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
else:
line = f"- [{category} | {confidence:.2f}] {content}"
line_text = ("\n" + line) if fact_lines else line
line_tokens = _count_tokens(line_text)
if running_tokens + line_tokens <= max_tokens:
fact_lines.append(line)
running_tokens += line_tokens
else:
break
if fact_lines:
sections.append("Facts:\n" + "\n".join(fact_lines))
if not sections:
return ""
result = "\n\n".join(sections)
token_count = _count_tokens(result)
if token_count > max_tokens:
char_per_token = len(result) / token_count
target_chars = int(max_tokens * char_per_token * 0.95)
result = result[:target_chars] + "\n..."
return result
def format_conversation_for_update(messages: list[Any]) -> str:
"""Format conversation messages for memory update prompt.
Args:
messages: List of conversation messages.
Returns:
Formatted conversation string.
"""
lines = []
for msg in messages:
role = getattr(msg, "type", "unknown")
content = getattr(msg, "content", str(msg))
# Handle content that might be a list (multimodal)
if isinstance(content, list):
text_parts = []
for p in content:
if isinstance(p, str):
text_parts.append(p)
elif isinstance(p, dict):
text_val = p.get("text")
if isinstance(text_val, str):
text_parts.append(text_val)
content = " ".join(text_parts) if text_parts else str(content)
# Strip uploaded_files tags from human messages to avoid persisting
# ephemeral file path info into long-term memory. Skip the turn entirely
# when nothing remains after stripping (upload-only message).
if role == "human":
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
if not content:
continue
# Truncate very long messages
if len(str(content)) > 1000:
content = str(content)[:1000] + "..."
if role == "human":
lines.append(f"User: {content}")
elif role == "ai":
lines.append(f"Assistant: {content}")
return "\n\n".join(lines)
__all__ = [
"MEMORY_UPDATE_PROMPT",
"FACT_EXTRACTION_PROMPT",
"format_memory_for_injection",
"format_conversation_for_update",
"_count_tokens",
"_coerce_confidence",
]
@@ -0,0 +1,253 @@
"""Memory V1 — 防抖异步更新队列(原版 DeerFlow MemoryUpdateQueue)。"""
from __future__ import annotations
import logging
import threading
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
logger = logging.getLogger(__name__)
def _get_v1_cfg():
from deerflow.config.memory_config import get_memory_config
return get_memory_config().v1
def _memory_enabled() -> bool:
from deerflow.config.memory_config import get_memory_config
return get_memory_config().enabled
@dataclass
class ConversationContext:
"""Context for a conversation to be processed for memory update."""
thread_id: str
messages: list[Any]
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
agent_name: str | None = None
user_id: str | None = None
correction_detected: bool = False
reinforcement_detected: bool = False
class MemoryUpdateQueue:
"""Queue for memory updates with debounce mechanism.
Collects conversation contexts and processes them after a configurable
debounce period. Multiple conversations received within the debounce
window are deduplicated by (thread_id, user_id, agent_name).
"""
def __init__(self) -> None:
self._queue: list[ConversationContext] = []
self._lock = threading.Lock()
self._timer: threading.Timer | None = None
self._processing = False
@staticmethod
def _queue_key(thread_id: str, user_id: str | None, agent_name: str | None) -> tuple[str, str | None, str | None]:
return (thread_id, user_id, agent_name)
def add(
self,
thread_id: str,
messages: list[Any],
agent_name: str | None = None,
user_id: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
) -> None:
"""Add a conversation to the update queue (with debounce)."""
if not _memory_enabled():
return
with self._lock:
self._enqueue_locked(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
self._reset_timer()
logger.info("V1 memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
def add_nowait(
self,
thread_id: str,
messages: list[Any],
agent_name: str | None = None,
user_id: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
) -> None:
"""Add a conversation and start processing immediately in the background."""
if not _memory_enabled():
return
with self._lock:
self._enqueue_locked(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
self._schedule_timer(0)
logger.info("V1 memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
def _enqueue_locked(
self,
*,
thread_id: str,
messages: list[Any],
agent_name: str | None,
user_id: str | None,
correction_detected: bool,
reinforcement_detected: bool,
) -> None:
queue_key = self._queue_key(thread_id, user_id, agent_name)
existing = next(
(ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) == queue_key),
None,
)
merged_correction = correction_detected or (existing.correction_detected if existing is not None else False)
merged_reinforcement = reinforcement_detected or (existing.reinforcement_detected if existing is not None else False)
context = ConversationContext(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=merged_correction,
reinforcement_detected=merged_reinforcement,
)
self._queue = [ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) != queue_key]
self._queue.append(context)
def _reset_timer(self) -> None:
cfg = _get_v1_cfg()
self._schedule_timer(cfg.debounce_seconds)
logger.debug("V1 memory update timer set for %ss", cfg.debounce_seconds)
def _schedule_timer(self, delay_seconds: float) -> None:
if self._timer is not None:
self._timer.cancel()
self._timer = threading.Timer(delay_seconds, self._process_queue)
self._timer.daemon = True
self._timer.start()
def _process_queue(self) -> None:
from deerflow.agents.memory.v1.updater import MemoryUpdater
with self._lock:
if self._processing:
self._schedule_timer(0)
return
if not self._queue:
return
self._processing = True
contexts_to_process = self._queue.copy()
self._queue.clear()
self._timer = None
logger.info("V1 memory: processing %d queued updates", len(contexts_to_process))
try:
updater = MemoryUpdater()
for context in contexts_to_process:
try:
logger.info("V1 memory: updating for thread %s", context.thread_id)
success = updater.update_memory(
messages=context.messages,
thread_id=context.thread_id,
agent_name=context.agent_name,
correction_detected=context.correction_detected,
reinforcement_detected=context.reinforcement_detected,
user_id=context.user_id,
)
if success:
logger.info("V1 memory: updated successfully for thread %s", context.thread_id)
else:
logger.warning("V1 memory: update skipped/failed for thread %s", context.thread_id)
except Exception as e: # noqa: BLE001
logger.error("V1 memory: error updating for thread %s: %s", context.thread_id, e)
if len(contexts_to_process) > 1:
time.sleep(0.5)
finally:
with self._lock:
self._processing = False
def flush(self) -> None:
"""Force immediate processing of the queue."""
with self._lock:
if self._timer is not None:
self._timer.cancel()
self._timer = None
self._process_queue()
def flush_nowait(self) -> None:
"""Start queue processing immediately in a background thread."""
with self._lock:
self._schedule_timer(0)
def clear(self) -> None:
"""Clear the queue without processing (for testing)."""
with self._lock:
if self._timer is not None:
self._timer.cancel()
self._timer = None
self._queue.clear()
self._processing = False
@property
def pending_count(self) -> int:
with self._lock:
return len(self._queue)
@property
def is_processing(self) -> bool:
with self._lock:
return self._processing
_memory_queue: MemoryUpdateQueue | None = None
_queue_lock = threading.Lock()
def get_memory_queue() -> MemoryUpdateQueue:
"""Get the global memory update queue singleton."""
global _memory_queue
with _queue_lock:
if _memory_queue is None:
_memory_queue = MemoryUpdateQueue()
return _memory_queue
def reset_memory_queue() -> None:
"""Reset the global memory queue (for testing)."""
global _memory_queue
with _queue_lock:
if _memory_queue is not None:
_memory_queue.clear()
_memory_queue = None
__all__ = [
"ConversationContext",
"MemoryUpdateQueue",
"get_memory_queue",
"reset_memory_queue",
]
@@ -0,0 +1,226 @@
"""Memory V1 — 存储层(原版 DeerFlow 格式)。
JSON 结构:
{
"version": "1.0",
"lastUpdated": "...",
"user": {
"workContext": {"summary": "", "updatedAt": ""},
"personalContext": {"summary": "", "updatedAt": ""},
"topOfMind": {"summary": "", "updatedAt": ""}
},
"history": {
"recentMonths": {"summary": "", "updatedAt": ""},
"earlierContext": {"summary": "", "updatedAt": ""},
"longTermBackground":{"summary": "", "updatedAt": ""}
},
"facts": []
}
"""
from __future__ import annotations
import json
import logging
import threading
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from deerflow.config.agents_config import AGENT_NAME_PATTERN
from deerflow.config.paths import get_paths
logger = logging.getLogger(__name__)
def utc_now_iso_z() -> str:
"""Current UTC time as ISO-8601 with ``Z`` suffix."""
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
def create_empty_memory() -> dict[str, Any]:
"""Create an empty memory structure (new format)."""
return {
"version": "1.0",
"lastUpdated": utc_now_iso_z(),
"user": {
"workContext": {"summary": "", "updatedAt": ""},
"personalContext": {"summary": "", "updatedAt": ""},
"topOfMind": {"summary": "", "updatedAt": ""},
},
"history": {
"recentMonths": {"summary": "", "updatedAt": ""},
"earlierContext": {"summary": "", "updatedAt": ""},
"longTermBackground": {"summary": "", "updatedAt": ""},
},
"facts": [],
}
def _migrate_legacy(data: dict[str, Any]) -> dict[str, Any]:
"""把老格式 {userContext: {workContext: str}} 迁移到新格式 {user: {workContext: {summary, updatedAt}}}。"""
if "userContext" not in data:
return data
now = utc_now_iso_z()
old_ctx = data.pop("userContext", {})
data["user"] = {
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
for k, v in old_ctx.items()
}
# history 字段可能是纯字符串,需要包裹
if "history" in data:
hist = data["history"]
data["history"] = {
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
for k, v in hist.items()
}
# 确保 user 包含所有必要字段
base = create_empty_memory()
for key in ("user", "history"):
section = base[key]
for field in section:
if field not in data.get(key, {}):
data.setdefault(key, {})[field] = section[field]
data["version"] = "1.0"
data.pop("updatedAt", None)
return data
class FileMemoryStorage:
"""File-based memory storage provider (原版 DeerFlow 实现)。"""
def __init__(self) -> None:
# Per-(user_id, agent_name) cache: value = (memory_data, file_mtime)
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
self._cache_lock = threading.Lock()
def _validate_agent_name(self, agent_name: str) -> None:
if not agent_name:
raise ValueError("Agent name must be a non-empty string.")
if not AGENT_NAME_PATTERN.match(agent_name):
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
paths = get_paths()
if user_id is not None:
if agent_name is not None:
self._validate_agent_name(agent_name)
return paths.user_agent_memory_file(user_id, agent_name)
return paths.user_memory_file(user_id)
# Legacy: no user_id
if agent_name is not None:
self._validate_agent_name(agent_name)
return paths.agent_memory_file(agent_name)
return paths.memory_file
def _load_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
if not file_path.exists():
return create_empty_memory()
try:
with open(file_path, encoding="utf-8") as f:
data = json.load(f)
if not isinstance(data, dict):
return create_empty_memory()
data = _migrate_legacy(data)
return data
except (json.JSONDecodeError, OSError) as e:
logger.warning("Failed to load memory file %s: %s", file_path, e)
return create_empty_memory()
@staticmethod
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
return (user_id, agent_name)
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Load memory data (cached with file modification time check)."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
except OSError:
current_mtime = None
with self._cache_lock:
cached = self._memory_cache.get(cache_key)
if cached is not None and cached[1] == current_mtime:
return cached[0]
memory_data = self._load_from_file(agent_name, user_id=user_id)
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, current_mtime)
return memory_data
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Reload memory data from file, forcing cache invalidation."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
memory_data = self._load_from_file(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
mtime = file_path.stat().st_mtime if file_path.exists() else None
except OSError:
mtime = None
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, mtime)
return memory_data
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
"""Save memory data to file and update cache (atomic write)."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
file_path.parent.mkdir(parents=True, exist_ok=True)
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
with open(temp_path, "w", encoding="utf-8") as f:
json.dump(memory_data, f, indent=2, ensure_ascii=False)
temp_path.replace(file_path)
try:
mtime = file_path.stat().st_mtime
except OSError:
mtime = None
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, mtime)
logger.info("Memory saved to %s", file_path)
return True
except OSError as e:
logger.error("Failed to save memory file: %s", e)
return False
_storage_instance: FileMemoryStorage | None = None
_storage_lock = threading.Lock()
def get_memory_storage() -> FileMemoryStorage:
"""Get the global FileMemoryStorage singleton."""
global _storage_instance
if _storage_instance is not None:
return _storage_instance
with _storage_lock:
if _storage_instance is None:
_storage_instance = FileMemoryStorage()
return _storage_instance
__all__ = [
"create_empty_memory",
"utc_now_iso_z",
"FileMemoryStorage",
"get_memory_storage",
]
@@ -0,0 +1,55 @@
"""Hooks fired before summarization removes messages from state(原版 DeerFlow)。"""
from __future__ import annotations
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
from deerflow.agents.memory.v1.queue import get_memory_queue
from deerflow.config.memory_config import get_memory_config
def memory_flush_hook(event: object) -> None:
"""Flush messages about to be summarized into the V1 memory queue.
``event`` is expected to be a ``SummarizationEvent``-like object with:
- ``thread_id: str``
- ``messages_to_summarize: list``
- ``agent_name: str | None``
- ``runtime: RunnableConfig``
"""
if not get_memory_config().enabled:
return
thread_id = getattr(event, "thread_id", None)
if not thread_id:
return
messages_to_summarize = getattr(event, "messages_to_summarize", [])
filtered_messages = filter_messages_for_memory(list(messages_to_summarize))
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
if not user_messages or not assistant_messages:
return
correction_detected = detect_correction(filtered_messages)
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
try:
from deerflow.runtime.user_context import resolve_runtime_user_id
runtime = getattr(event, "runtime", None)
user_id = resolve_runtime_user_id(runtime) if runtime is not None else None
except Exception: # noqa: BLE001
user_id = None
agent_name = getattr(event, "agent_name", None)
queue = get_memory_queue()
queue.add_nowait(
thread_id=thread_id,
messages=filtered_messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
__all__ = ["memory_flush_hook"]
@@ -0,0 +1,495 @@
"""Memory V1 — updater(原版 DeerFlow MemoryUpdater)。"""
from __future__ import annotations
import asyncio
import atexit
import concurrent.futures
import copy
import json
import logging
import math
import re
import uuid
from typing import Any
from deerflow.agents.memory.v1.prompt import (
MEMORY_UPDATE_PROMPT,
format_conversation_for_update,
)
from deerflow.agents.memory.v1.storage import (
create_empty_memory,
get_memory_storage,
utc_now_iso_z,
)
logger = logging.getLogger(__name__)
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=4,
thread_name_prefix="v1-memory-updater-sync",
)
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
def _get_v1_cfg():
"""返回 V1MemoryConfig(含 max_facts, fact_confidence_threshold, model_name)。"""
from deerflow.config.memory_config import get_memory_config
return get_memory_config().v1
# ---------------------------------------------------------------------------
# 向后兼容的便捷函数(供 gateway router 等调用)
# ---------------------------------------------------------------------------
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
return get_memory_storage().load(agent_name, user_id=user_id)
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
return get_memory_storage().reload(agent_name, user_id=user_id)
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
storage = get_memory_storage()
if not storage.save(memory_data, agent_name, user_id=user_id):
raise OSError("Failed to save imported memory data")
return storage.load(agent_name, user_id=user_id)
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
cleared = create_empty_memory()
if not get_memory_storage().save(cleared, agent_name, user_id=user_id):
raise OSError("Failed to save cleared memory data")
return cleared
def _validate_confidence(confidence: float) -> float:
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
raise ValueError("confidence")
return confidence
def create_memory_fact(
content: str,
category: str = "context",
confidence: float = 0.5,
agent_name: str | None = None,
*,
user_id: str | None = None,
) -> dict[str, Any]:
normalized_content = content.strip()
if not normalized_content:
raise ValueError("content")
normalized_category = category.strip() or "context"
validated_confidence = _validate_confidence(confidence)
now = utc_now_iso_z()
memory_data = get_memory_data(agent_name, user_id=user_id)
updated_memory = dict(memory_data)
facts = list(memory_data.get("facts", []))
facts.append({
"id": f"fact_{uuid.uuid4().hex[:8]}",
"content": normalized_content,
"category": normalized_category,
"confidence": validated_confidence,
"createdAt": now,
"source": "manual",
})
updated_memory["facts"] = facts
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
raise OSError("Failed to save memory data after creating fact")
return updated_memory
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
memory_data = get_memory_data(agent_name, user_id=user_id)
facts = memory_data.get("facts", [])
updated_facts = [f for f in facts if f.get("id") != fact_id]
if len(updated_facts) == len(facts):
raise KeyError(fact_id)
updated_memory = dict(memory_data)
updated_memory["facts"] = updated_facts
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
return updated_memory
def update_memory_fact(
fact_id: str,
content: str | None = None,
category: str | None = None,
confidence: float | None = None,
agent_name: str | None = None,
*,
user_id: str | None = None,
) -> dict[str, Any]:
memory_data = get_memory_data(agent_name, user_id=user_id)
updated_memory = dict(memory_data)
updated_facts: list[dict[str, Any]] = []
found = False
for fact in memory_data.get("facts", []):
if fact.get("id") == fact_id:
found = True
updated_fact = dict(fact)
if content is not None:
normalized = content.strip()
if not normalized:
raise ValueError("content")
updated_fact["content"] = normalized
if category is not None:
updated_fact["category"] = category.strip() or "context"
if confidence is not None:
updated_fact["confidence"] = _validate_confidence(confidence)
updated_facts.append(updated_fact)
else:
updated_facts.append(fact)
if not found:
raise KeyError(fact_id)
updated_memory["facts"] = updated_facts
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
return updated_memory
# ---------------------------------------------------------------------------
# 内容提取辅助
# ---------------------------------------------------------------------------
def _extract_text(content: Any) -> str:
"""从 LLM response content(str 或 list of blocks)提取纯文本。"""
if isinstance(content, str):
return content
if isinstance(content, list):
pieces: list[str] = []
pending: list[str] = []
def flush():
if pending:
pieces.append("".join(pending))
pending.clear()
for block in content:
if isinstance(block, str):
pending.append(block)
elif isinstance(block, dict):
flush()
text_val = block.get("text")
if isinstance(text_val, str):
pieces.append(text_val)
flush()
return "\n".join(pieces)
return str(content)
_UPLOAD_SENTENCE_RE = re.compile(
r"[^.!?]*\b(?:"
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
r"|file\s+upload"
r"|/mnt/user-data/uploads/"
r"|<uploaded_files>"
r")[^.!?]*[.!?]?\s*",
re.IGNORECASE,
)
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
"""移除记忆中关于文件上传事件的句子(上传文件是会话级临时的)。"""
for section in ("user", "history"):
section_data = memory_data.get(section, {})
for _key, val in section_data.items():
if isinstance(val, dict) and "summary" in val:
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
cleaned = re.sub(r" +", " ", cleaned)
val["summary"] = cleaned
facts = memory_data.get("facts", [])
if facts:
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
return memory_data
def _fact_content_key(content: Any) -> str | None:
if not isinstance(content, str):
return None
stripped = content.strip()
if not stripped:
return None
return stripped.casefold()
# ---------------------------------------------------------------------------
# MemoryUpdater(原版核心类)
# ---------------------------------------------------------------------------
class MemoryUpdater:
"""Updates memory using LLM based on conversation context."""
def __init__(self, model_name: str | None = None) -> None:
self._model_name = model_name
def _get_model(self):
from deerflow.models import create_chat_model
cfg = _get_v1_cfg()
model_name = self._model_name or cfg.model_name
return create_chat_model(name=model_name or None, thinking_enabled=False)
def _build_correction_hint(self, correction_detected: bool, reinforcement_detected: bool) -> str:
correction_hint = ""
if correction_detected:
correction_hint = (
"IMPORTANT: Explicit correction signals were detected in this conversation. "
"Pay special attention to what the agent got wrong, what the user corrected, "
"and record the correct approach as a fact with category "
'"correction" and confidence >= 0.95 when appropriate.'
)
if reinforcement_detected:
reinforcement_hint = (
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
"The user explicitly confirmed the agent's approach was correct or helpful. "
"Record the confirmed approach, style, or preference as a fact with category "
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
)
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
return correction_hint
def _prepare_update_prompt(
self,
messages: list[Any],
agent_name: str | None,
correction_detected: bool,
reinforcement_detected: bool,
user_id: str | None = None,
) -> tuple[dict[str, Any], str] | None:
from deerflow.config.memory_config import get_memory_config
config = get_memory_config()
if not config.enabled or not messages:
return None
current_memory = get_memory_data(agent_name, user_id=user_id)
conversation_text = format_conversation_for_update(messages)
if not conversation_text.strip():
return None
correction_hint = self._build_correction_hint(correction_detected, reinforcement_detected)
prompt = MEMORY_UPDATE_PROMPT.format(
current_memory=json.dumps(current_memory, indent=2, ensure_ascii=False),
conversation=conversation_text,
correction_hint=correction_hint,
)
return current_memory, prompt
def _finalize_update(
self,
current_memory: dict[str, Any],
response_content: Any,
thread_id: str | None,
agent_name: str | None,
user_id: str | None = None,
) -> bool:
response_text = _extract_text(response_content).strip()
if response_text.startswith("```"):
lines = response_text.split("\n")
response_text = "\n".join(lines[1:-1] if lines[-1] == "```" else lines[1:])
update_data = json.loads(response_text)
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
async def aupdate_memory(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
return await asyncio.to_thread(
self._do_update_memory_sync,
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
def _do_update_memory_sync(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
try:
prepared = self._prepare_update_prompt(
messages=messages,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
if prepared is None:
return False
current_memory, prompt = prepared
model = self._get_model()
response = model.invoke(prompt, config={"run_name": "v1_memory_agent"})
return self._finalize_update(
current_memory=current_memory,
response_content=response.content,
thread_id=thread_id,
agent_name=agent_name,
user_id=user_id,
)
except json.JSONDecodeError as e:
logger.warning("V1 memory: failed to parse LLM response: %s", e)
return False
except Exception as e: # noqa: BLE001
logger.exception("V1 memory update failed: %s", e)
return False
def update_memory(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
try:
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
self._do_update_memory_sync,
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
return future.result()
except Exception: # noqa: BLE001
logger.exception("V1 memory: failed to offload update to executor")
return False
return self._do_update_memory_sync(
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
def _apply_updates(
self,
current_memory: dict[str, Any],
update_data: dict[str, Any],
thread_id: str | None = None,
) -> dict[str, Any]:
cfg = _get_v1_cfg()
now = utc_now_iso_z()
# 更新 user context 段
user_updates = update_data.get("user", {})
for section in ("workContext", "personalContext", "topOfMind"):
section_data = user_updates.get(section, {})
if section_data.get("shouldUpdate") and section_data.get("summary"):
current_memory.setdefault("user", {})[section] = {
"summary": section_data["summary"],
"updatedAt": now,
}
# 更新 history 段
history_updates = update_data.get("history", {})
for section in ("recentMonths", "earlierContext", "longTermBackground"):
section_data = history_updates.get(section, {})
if section_data.get("shouldUpdate") and section_data.get("summary"):
current_memory.setdefault("history", {})[section] = {
"summary": section_data["summary"],
"updatedAt": now,
}
# 删除事实
facts_to_remove = set(update_data.get("factsToRemove", []))
if facts_to_remove:
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
# 新增事实(去重 + 置信度过滤)
existing_keys = {
key for key in (_fact_content_key(f.get("content")) for f in current_memory.get("facts", []))
if key is not None
}
for fact in update_data.get("newFacts", []):
confidence = fact.get("confidence", 0.5)
if confidence < cfg.fact_confidence_threshold:
continue
raw_content = fact.get("content", "")
if not isinstance(raw_content, str):
continue
normalized_content = raw_content.strip()
fact_key = _fact_content_key(normalized_content)
if fact_key is not None and fact_key in existing_keys:
continue
fact_entry: dict[str, Any] = {
"id": f"fact_{uuid.uuid4().hex[:8]}",
"content": normalized_content,
"category": fact.get("category", "context"),
"confidence": confidence,
"createdAt": now,
"source": thread_id or "unknown",
}
source_error = fact.get("sourceError")
if isinstance(source_error, str) and source_error.strip():
fact_entry["sourceError"] = source_error.strip()
current_memory["facts"].append(fact_entry)
if fact_key is not None:
existing_keys.add(fact_key)
# 超出上限时驱逐置信度最低的事实
if len(current_memory["facts"]) > cfg.max_facts:
current_memory["facts"] = sorted(
current_memory["facts"],
key=lambda f: f.get("confidence", 0),
reverse=True,
)[:cfg.max_facts]
return current_memory
def update_memory_from_conversation(
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
updater = MemoryUpdater()
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
__all__ = [
"MemoryUpdater",
"update_memory_from_conversation",
"get_memory_data",
"reload_memory_data",
"import_memory_data",
"clear_memory_data",
"create_memory_fact",
"delete_memory_fact",
"update_memory_fact",
]
@@ -171,21 +171,21 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
seconds = max(1, round(wait_ms / 1000))
reason_text = "provider is busy" if reason == "busy" else "provider request failed temporarily"
return f"LLM request retry {attempt}/{self.retry_max_attempts}: {reason_text}. Retrying in {seconds}s."
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
def _build_circuit_breaker_message(self) -> str:
return "The configured LLM provider is currently unavailable due to continuous failures. Circuit breaker is engaged to protect the system. Please wait a moment before trying again."
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
def _build_user_message(self, exc: BaseException, reason: str) -> str:
detail = _extract_error_detail(exc)
if reason == "quota":
return "The configured LLM provider rejected the request because the account is out of quota, billing is unavailable, or usage is restricted. Please fix the provider account and try again."
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
if reason == "auth":
return "The configured LLM provider rejected the request because authentication or access is invalid. Please check the provider credentials and try again."
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
if reason in {"busy", "transient"}:
return "The configured LLM provider is temporarily unavailable after multiple retries. Please wait a moment and continue the conversation."
return f"LLM request failed: {detail}"
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
return f"大模型因「{detail}」原因调用失败。"
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
try:
@@ -90,16 +90,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
state_schema = MemoryMiddlewareState
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None, injection_enabled: bool = True):
"""初始化 MemoryMiddleware。
Args:
agent_name: 提供时记忆按 agent 隔离;None 用默认 agent。
memory_config: 显式记忆配置;省略时用全局配置。
injection_enabled: 运行时开关;``False`` 时跳过 Hindsight 动态召回注入。
"""
super().__init__()
self._agent_name = agent_name
self._memory_config = memory_config
self._injection_enabled = injection_enabled
# ---- 运行时上下文解析 -------------------------------------------------
@@ -115,15 +117,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
except RuntimeError:
return None
def _resolve_agent_id(self, runtime: Runtime | None) -> str:
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。"""
def _resolve_agent_id(self, runtime: Runtime | None) -> str | None:
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。
无 agent 上下文时返回 None,使 V1 记忆存入全局文件(users/{id}/memory.json)。
"""
if self._agent_name:
return self._agent_name
if runtime is not None and getattr(runtime, "context", None):
agent_id = runtime.context.get("agent_id")
if agent_id:
return str(agent_id)
return "default"
return None
# ---- wrap_model_call:Hindsight 动态召回注入 -------------------------
@@ -133,6 +138,8 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
背景信息而非用户的新输入。
"""
if not self._injection_enabled:
return None
config = self._memory_config or get_memory_config()
if not config.enabled:
return None
@@ -0,0 +1,158 @@
"""Middleware for intercepting skill calls and stopping execution when specific skills are invoked."""
import logging
import re
from collections.abc import Callable
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain_core.messages import ToolMessage
from langgraph.prebuilt.tool_node import ToolCallRequest
from langgraph.types import Command, interrupt
logger = logging.getLogger(__name__)
def extract_name_task(text: str) -> tuple[str, str]:
"""Extract `agent_name` and `task_content` from a bash invocation that triggers
a sub-agent via ``python3 scripts/agent_orchestration.py "<agent>" "<task>"``.
Returns empty strings when no match is found so the caller can treat the
absence of a match as "not a target skill call".
"""
try:
pattern = r'\.py\s+"([^"]*)"\s+"([^"]*)"'
match = re.search(pattern, text)
if match:
return match.group(1), match.group(2)
return "", ""
except Exception:
return "", ""
class SkillStopMiddlewareState(AgentState):
"""Compatible with the `ThreadState` schema."""
pass
class SkillStopMiddleware(AgentMiddleware[SkillStopMiddlewareState]):
"""Intercepts skill file reads and stops execution when specific skills are invoked.
When the model reads a file from a skill directory that matches one of the
configured skill names, this middleware:
1. Intercepts the tool call before execution
2. Returns a Command that stops execution (goes to END)
3. Optionally returns a message indicating which skill was triggered
This is useful for implementing "skill triggers" that pause the agent
when certain skills are about to be used.
"""
state_schema = SkillStopMiddlewareState
def __init__(
self,
*,
skill_names: list[str],
skills_root: str = "/mnt/skills",
stop_message: str | None = None,
stop_on_match: bool = True,
):
"""Initialize the skill stop middleware.
Args:
skill_names: List of skill names to intercept (e.g., ["newsletter-generation", "podcast-generation"]).
skills_root: Root directory where skills are located (default: "/mnt/skills").
stop_message: Optional message to return when stopping. If None, a default message is used.
stop_on_match: If True, stop when skill matches. If False, continue execution (for logging only).
"""
super().__init__()
self._skill_names = set(skill_names)
self._skills_root = skills_root.rstrip("/")
self._stop_message = stop_message
self._stop_on_match = stop_on_match
def _is_target_skill_call(self, tool_call: dict[str, Any]) -> str | None:
"""Return the matched skill name if this tool call invokes one of the
configured target skills, otherwise ``None``.
Two invocation patterns are recognized:
1. **Structured tool call** — the model emits a tool call whose ``name``
literally matches one of the configured skill names (e.g. the
multi-agent leader calling
``agent_orchestration({"agent_name": "...", "task": "..."})``).
This is the natural form most LLMs produce when told to "use a
skill", and it's the form that drives the roundtable dispatch.
2. **Bash + argv** — the legacy form where the lead agent reaches into
a skill by running ``python3 scripts/<skill-name>.py "<arg1>" "<arg2>"``.
Kept for backward compatibility with skills that use bash for
dispatch.
"""
name = tool_call.get("name") or ""
lowered_targets = {s.lower() for s in self._skill_names}
# Pattern 1: structured tool call whose tool name IS the skill name.
if name and (name in self._skill_names or name.lower() in lowered_targets):
logger.info("SkillStopMiddleware: matched structured skill call name=%s", name)
return name
# Pattern 2: bash invocation with argv-style skill name.
if name == "bash" and "args" in tool_call:
skill_name, _ = extract_name_task(str(tool_call["args"]))
if skill_name and (skill_name in self._skill_names or skill_name.lower() in lowered_targets):
logger.info("SkillStopMiddleware: matched bash skill call name=%s", skill_name)
return skill_name
return None
def _handle_skill_stop(self, request: ToolCallRequest, skill_name: str) -> Command:
"""Build the stop response — an interrupt that includes a ToolMessage
describing which skill was triggered."""
logger.info("Skill stop triggered: skill_name=%s", skill_name)
tool_call_id = request.tool_call.get("id", "")
message_content = self._stop_message or f"⏹️ Skill '{skill_name}' triggered. Execution stopped."
tool_message = ToolMessage(
id=f"skill_stop:{tool_call_id}",
content=message_content,
tool_call_id=tool_call_id,
name="skill_stop",
)
return interrupt(
{
"messages": [tool_message],
"skill_name": skill_name,
}
)
@override
def wrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], ToolMessage | Command],
) -> ToolMessage | Command:
matched_skill = self._is_target_skill_call(request.tool_call)
if matched_skill:
if self._stop_on_match:
return self._handle_skill_stop(request, matched_skill)
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
return handler(request)
@override
async def awrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], ToolMessage | Command],
) -> ToolMessage | Command:
matched_skill = self._is_target_skill_call(request.tool_call)
if matched_skill:
if self._stop_on_match:
return self._handle_skill_stop(request, matched_skill)
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
return await handler(request)
@@ -105,8 +105,16 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
return prompt, user_msg
def _strip_think_tags(self, text: str) -> str:
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, DeepSeek-R1)."""
return re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE).strip()
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, GLM, DeepSeek-R1).
Also handles malformed cases: an unclosed `<think>` (response truncated mid-reasoning)
or a stray `</think>` (open tag eaten by upstream parsing) — both can otherwise leak
raw reasoning into the title.
"""
text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
text = re.sub(r"<think>[\s\S]*$", "", text, flags=re.IGNORECASE)
text = re.sub(r"^[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
return text.strip()
def _parse_title(self, content: object) -> str:
"""Normalize model output into a clean title string."""
@@ -146,6 +154,36 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
_, user_msg = self._build_title_prompt(state)
return {"title": self._fallback_title(user_msg)}
def _force_disable_thinking(self, model: object) -> None:
"""Force-disable thinking on the title model via `extra_body` overrides.
Title generation never needs reasoning. The model factory only emits
disable params when the model config declares `when_thinking_disabled`
or a matching `when_thinking_enabled` shape — many user-added reasoning
models (e.g. GLM-4.5/5) only set `supports_thinking: true` and would
otherwise keep thinking on, which is slow and leaks <think> tags into
the title prompt response. Injecting both common OpenAI-compatible
disable shapes covers Zhipu GLM (`thinking.type`) and Qwen/vLLM
(`chat_template_kwargs.enable_thinking`); other providers ignore
unknown body fields.
"""
if not hasattr(model, "extra_body"):
return
current = getattr(model, "extra_body", None) or {}
merged_chat_template_kwargs = {
**(current.get("chat_template_kwargs") or {}),
"enable_thinking": False,
}
merged = {
**current,
"thinking": {**(current.get("thinking") or {}), "type": "disabled"},
"chat_template_kwargs": merged_chat_template_kwargs,
}
try:
model.extra_body = merged
except Exception:
logger.debug("Could not override extra_body to disable thinking on title model", exc_info=True)
async def _agenerate_title_result(self, state: TitleMiddlewareState) -> dict | None:
"""Generate a title asynchronously and fall back locally on failure."""
if not self._should_generate_title(state):
@@ -162,6 +200,7 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
model = create_chat_model(name=config.model_name, **model_kwargs)
else:
model = create_chat_model(**model_kwargs)
self._force_disable_thinking(model)
response = await model.ainvoke(prompt, config=self._get_runnable_config())
title = self._parse_title(response.content)
if title:
@@ -57,11 +57,18 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
agent_id: The agent storage/runtime id.
Returns:
AgentConfig instance.
AgentConfig instance, or ``None`` when the on-disk artifacts are missing.
Raises:
FileNotFoundError: If the agent directory or config.yaml does not exist.
ValueError: If config.yaml cannot be parsed.
ValueError: If config.yaml exists but cannot be parsed.
Tolerance:
A missing agent directory or missing config.yaml is **not** an error —
the runtime simply falls back to default Lead Agent behavior for that
id. This keeps the chat session usable when the DB still has an agent
row but its files have been deleted (e.g. data dir moved between
machines, or filesystem wiped). The router layer is expected to
lazily restore the directory next time the agent is fetched via API.
"""
if agent_id is None:
@@ -72,10 +79,20 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
config_file = agent_dir / "config.yaml"
if not agent_dir.exists():
raise FileNotFoundError(f"Agent directory not found: {agent_dir}")
logger.warning(
"Agent directory missing for id '%s' (%s); falling back to default config",
agent_id,
agent_dir,
)
return None
if not config_file.exists() and not (agent_dir / SOUL_FILENAME).exists():
raise FileNotFoundError(f"Agent config not found: {config_file}")
logger.warning(
"Agent has no config.yaml or %s at %s; falling back to default config",
SOUL_FILENAME,
agent_dir,
)
return None
data: dict[str, Any] = {}
if config_file.exists():
@@ -22,6 +22,7 @@ from deerflow.config.sandbox_config import SandboxConfig
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
from deerflow.config.skill_evolution_runtime import get_overrides_mtime as _get_skill_evolution_overrides_mtime
from deerflow.config.skill_evolution_runtime import load_overrides as _load_skill_evolution_overrides
from deerflow.config.skill_stop_config import SkillStopConfig
from deerflow.config.skills_config import SkillsConfig
from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict
from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict
@@ -117,6 +118,8 @@ class AppConfig(BaseModel):
subagents: SubagentsAppConfig = Field(default_factory=SubagentsAppConfig, description="Subagent runtime configuration")
guardrails: GuardrailsConfig = Field(default_factory=GuardrailsConfig, description="Guardrail middleware configuration")
circuit_breaker: CircuitBreakerConfig = Field(default_factory=CircuitBreakerConfig, description="LLM circuit breaker configuration")
excluded_tools: list[str] = Field(default_factory=list, description="Tool names to exclude from the main agent (e.g. ['task', 'ask_clarification'])")
skill_stop: SkillStopConfig = Field(default_factory=SkillStopConfig, description="Skill stop middleware configuration")
model_config = ConfigDict(extra="allow")
database: DatabaseConfig = Field(default_factory=DatabaseConfig, description="Unified database backend configuration")
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
@@ -12,21 +12,21 @@
injection: {...} # 系统提示词注入
security: {...} # 内容安全扫描
三层配置覆盖
两层配置覆盖
============
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
4. Pydantic 默认值
1. per-user 覆盖 数据库 ``user_memory_configs`` 表 (经进程内 cache 读取)
(有 per-user 记录时自动强制 enabled=True)
2. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
3. Pydantic 默认值
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
第 1 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
cache 由 Gateway lifespan 启动时预热,写入时同步更新,保持同步读取路径。
"""
from __future__ import annotations
import copy
import json
import logging
from typing import Any
@@ -73,7 +73,7 @@ class HindsightMemoryConfig(BaseModel):
description="agent 私有工作 bank 命名模板",
)
memory_mode: str = Field(
default="context",
default="hybrid",
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
)
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
@@ -113,15 +113,30 @@ class SecurityConfig(BaseModel):
# ===========================================================================
class V1MemoryConfig(BaseModel):
"""原版 DeerFlow V1 记忆 provider 配置 (memory.json + LLM 自动提取)。"""
max_facts: int = Field(default=100, ge=10, le=500, description="最多保留的事实条数")
fact_confidence_threshold: float = Field(default=0.7, ge=0.0, le=1.0, description="事实置信度阈值,低于此值不保存")
debounce_seconds: int = Field(default=30, ge=5, le=300, description="对话结束后延迟多少秒才触发 LLM 提取")
model_name: str = Field(default="", description="用于事实提取的 LLM 模型名;空则继承全局默认模型")
token_budget: int = Field(default=2000, ge=200, le=8000, description="注入系统提示词的 token 预算")
class MemoryConfig(BaseModel):
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
enabled: bool = Field(default=True, description="记忆子系统总开关")
version: str = Field(
default="v2",
description="记忆系统版本:v1(原版 DeerFlow,memory.json + LLM 自动提取) 或 v2(USER.md/MEMORY.md)",
)
provider: str = Field(
default="builtin",
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
)
v1: V1MemoryConfig = Field(default_factory=V1MemoryConfig)
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
injection: InjectionConfig = Field(default_factory=InjectionConfig)
@@ -129,29 +144,26 @@ class MemoryConfig(BaseModel):
# ===========================================================================
# per-user 运行时覆盖
# per-user 运行时覆盖 — 白名单
# ===========================================================================
# 用户可在运行时覆盖的字段白名单 (点号路径)。
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
# 用户唯一可覆盖的字段:召回深度。
# injection.* / builtin.* / enabled 始终由 config.yaml 控制,用户不可关闭。
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
{
"injection.enabled",
"injection.max_tokens",
"injection.include_builtin",
"injection.include_hindsight",
"hindsight.memory_mode",
"hindsight.recall_budget",
"hindsight.bank_mission",
}
)
_VALID_RECALL_BUDGETS: frozenset[str] = frozenset({"low", "mid", "high"})
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
输入支持嵌套形式 ``{"hindsight": {"recall_budget": "high"}}`` 或
点号扁平形式 ``{"hindsight.recall_budget": "high"}``。
非白名单字段被丢弃并 log。
"""
result: dict[str, Any] = {}
@@ -182,21 +194,6 @@ def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any
return merged
def _load_json_file(path: Any) -> dict[str, Any]:
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
try:
from pathlib import Path
p = Path(path)
if not p.exists():
return {}
data = json.loads(p.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (OSError, json.JSONDecodeError) as exc:
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
return {}
# ===========================================================================
# 全局单例 + 加载入口
# ===========================================================================
@@ -221,85 +218,92 @@ def load_memory_config_from_dict(config_dict: dict) -> None:
_memory_config = MemoryConfig(**config_dict)
# ===========================================================================
# per-user recall_budget 进程内 cache
#
# Gateway lifespan 启动时调用 load_recall_budget_cache() 从 DB 批量加载。
# API 写入时同步更新 cache,使 get_effective_memory_config() 保持同步读取。
# ===========================================================================
_recall_budget_cache: dict[str, str] = {}
def load_recall_budget_cache(data: dict[str, str]) -> None:
"""用 DB 批量加载结果替换整个 cache (启动时调用一次)。"""
global _recall_budget_cache
_recall_budget_cache = dict(data)
logger.info("recall_budget cache loaded: %d user(s)", len(_recall_budget_cache))
def set_recall_budget_cache(user_id: str, value: str) -> None:
"""写入单个用户的 recall_budget cache。"""
_recall_budget_cache[user_id] = value
def delete_recall_budget_cache(user_id: str) -> None:
"""从 cache 中移除某用户的 recall_budget。"""
_recall_budget_cache.pop(user_id, None)
def get_recall_budget_from_cache(user_id: str) -> str | None:
"""返回 cache 中某用户的 recall_budget,未设置时返回 None。"""
return _recall_budget_cache.get(user_id)
def get_effective_memory_config(user_id: str) -> MemoryConfig:
"""按三层覆盖优先级,返回某用户的生效记忆配置。
"""返回某用户的生效记忆配置 (同步,从进程内 cache 读取 per-user 覆盖)。
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
优先级 (高 -> 低):per-user DB 覆盖 (经 cache) > config.yaml > 默认值。
有 per-user 记录时自动强制 enabled=True。
"""
base = _memory_config.model_dump()
recall_budget = _recall_budget_cache.get(user_id)
if recall_budget is not None:
base.setdefault("hindsight", {})["recall_budget"] = recall_budget
base["enabled"] = True
try:
from deerflow.config.paths import get_paths
paths = get_paths()
global_overrides = _filter_to_whitelist(
_load_json_file(paths.memory_global_overrides_file)
)
user_overrides = _filter_to_whitelist(
_load_json_file(paths.user_memory_config_file(user_id))
)
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
return _memory_config
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
try:
return MemoryConfig(**merged)
return MemoryConfig(**base)
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
return _memory_config
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
# ===========================================================================
# Gateway 调用的异步 DB 对接函数
# ===========================================================================
async def save_user_memory_override_db(user_id: str, patch: dict[str, Any], store: Any) -> dict[str, Any]:
"""把白名单字段写入 DB store 并更新 cache,返回生效配置 dict。
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
``store`` 为 ``UserMemoryConfigStore`` 实例 (避免循环 import,用 Any)。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
current = _load_json_file(path)
filtered = _filter_to_whitelist(patch)
merged = _deep_merge(current, filtered)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
return merged
recall_budget = (filtered.get("hindsight") or {}).get("recall_budget")
if recall_budget is not None:
if recall_budget not in _VALID_RECALL_BUDGETS:
raise ValueError(f"recall_budget 必须为 {sorted(_VALID_RECALL_BUDGETS)} 之一,收到:{recall_budget!r}")
await store.set_recall_budget(user_id, recall_budget)
set_recall_budget_cache(user_id, recall_budget)
return get_effective_memory_config(user_id).model_dump()
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
"""撤销某用户的 per-user 覆盖。
async def delete_user_memory_override_db(user_id: str, field: str | None, store: Any) -> dict[str, Any]:
"""撤销某用户的 per-user 覆盖并更新 cache,返回生效配置 dict。
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
``field=None`` 清空全部;``field="hindsight.recall_budget"`` 只清该字段。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
if field is None:
if path.exists():
path.unlink()
return {}
current = _load_json_file(path)
if "." in field:
section, sub_field = field.split(".", 1)
if isinstance(current.get(section), dict):
current[section].pop(sub_field, None)
if not current[section]:
current.pop(section, None)
if field is None or field == "hindsight.recall_budget":
await store.delete(user_id)
delete_recall_budget_cache(user_id)
else:
current.pop(field, None)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
return current
logger.warning("忽略删除非白名单的记忆配置覆盖字段: %s", field)
return get_effective_memory_config(user_id).model_dump()
__all__ = [
"V1MemoryConfig",
"BuiltinMemoryConfig",
"HindsightMemoryConfig",
"InjectionConfig",
@@ -310,6 +314,10 @@ __all__ = [
"set_memory_config",
"load_memory_config_from_dict",
"get_effective_memory_config",
"save_user_memory_override",
"delete_user_memory_override",
"load_recall_budget_cache",
"set_recall_budget_cache",
"delete_recall_budget_cache",
"get_recall_budget_from_cache",
"save_user_memory_override_db",
"delete_user_memory_override_db",
]
@@ -0,0 +1,31 @@
"""Configuration for skill stop middleware."""
from pydantic import BaseModel, Field
class SkillStopConfig(BaseModel):
"""Configuration for the skill stop middleware.
When enabled, the middleware will intercept skill invocations and stop
execution when a matching skill is about to be used.
"""
enabled: bool = Field(
default=False,
description="Enable skill stop middleware. When True, execution will stop when specified skills are invoked.",
)
skill_names: list[str] = Field(
default_factory=list,
description="List of skill names to intercept. When the model reads a file from these skill directories, execution will stop.",
)
stop_message: str | None = Field(
default=None,
description="Custom message to display when stopping. If None, a default message is used.",
)
stop_on_match: bool = Field(
default=True,
description="If True, stop execution when skill matches. If False, only log the match without stopping.",
)
@@ -6,6 +6,18 @@ from pydantic import BaseModel, Field
from deerflow.config.runtime_paths import project_root, resolve_path
class SecurityScanConfig(BaseModel):
"""Toggle for the LLM-based skill security scanner."""
enabled: bool = Field(
default=True,
description=(
"When false, skip the LLM security scan applied to skill installs, "
"custom-skill edits, and rollbacks. The scan is then treated as 'allow'."
),
)
class SkillsConfig(BaseModel):
"""Configuration for skills system"""
@@ -21,6 +33,10 @@ class SkillsConfig(BaseModel):
default="/mnt/skills",
description="Path where skills are mounted in the sandbox container",
)
security_scan: SecurityScanConfig = Field(
default_factory=SecurityScanConfig,
description="LLM security-scan toggle for skill content.",
)
def get_skills_path(self) -> Path:
"""
@@ -61,3 +61,23 @@ class AgentStore(ABC):
async def count_owned(self, user_id: str) -> int:
"""Return the number of agents owned by ``user_id`` (user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
"""Set or clear the homepage-selector featured rank for an agent.
Returns the updated row, or ``None`` if the agent does not exist.
Caller is responsible for authorization (admin) and for verifying
that ``published`` is true when ``order`` is not ``None``.
"""
raise NotImplementedError
@abstractmethod
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
raise NotImplementedError
@abstractmethod
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
raise NotImplementedError
@@ -55,6 +55,7 @@ class MemoryAgentStore(AgentStore):
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
@@ -70,6 +71,7 @@ class MemoryAgentStore(AgentStore):
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
@@ -117,3 +119,22 @@ class MemoryAgentStore(AgentStore):
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String
from sqlalchemy import BigInteger, Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -21,5 +21,11 @@ class AgentRow(Base):
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
# frontend can seed initial ranks from ``Date.now()`` (ms-resolution
# timestamps overflow a 4-byte INT on MySQL).
featured_order: Mapped[int | None] = mapped_column(BigInteger, nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -142,3 +142,35 @@ class AgentRepository(AgentStore):
stmt = select(func.count()).select_from(AgentRow).where(AgentRow.user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return None
row.featured_order = order
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
stmt = (
select(AgentRow)
.where(AgentRow.featured_order.is_not(None), self._visible_filter(user_id))
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
if not agent_ids:
return []
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._visible_filter(user_id))
async with self._sf() as session:
result = await session.execute(stmt)
rows = list(result.scalars())
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
return [_row_to_dict(r) for r in rows]
@@ -8,4 +8,39 @@ def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository
return AIWritingSessionRepository(session_factory)
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
# ── 进程级默认 repo 单例 ──────────────────────────────────────────────────────
#
# AI 写作 graph 注册成 LangGraph Server assistant 后,节点跑在 harness 进程里,
# 拿不到 FastAPI 的 ``request.app.state``;harness → app 又是禁止的 import 方向。
# 这里提供一个 app 启动时注册、harness 节点运行时读取的「跨边界单例」通道。
#
# 写入:app/gateway/deps.py 在创建完 session_factory 之后调
# ``register_default_store(app.state.ai_writing_session_store)``
# 读取:节点里 ``get_default_store()``,None 表示未配置 DB(跳过写入即可)。
_default_store: AIWritingSessionRepository | None = None
def register_default_store(store: AIWritingSessionRepository | None) -> None:
"""登记进程级默认 AI 写作 session store。
幂等:可多次调用,覆盖前值。传 ``None`` 用于显式清理(主要给测试)。
"""
global _default_store
_default_store = store
def get_default_store() -> AIWritingSessionRepository | None:
"""返回进程级默认 AI 写作 session store;未配置时为 ``None``。
节点调用方应在拿到 ``None`` 时跳过 DB 写入(写入是「锦上添花」非致命)。
"""
return _default_store
__all__ = [
"AIWritingSessionRow",
"AIWritingSessionRepository",
"make_ai_writing_session_store",
"register_default_store",
"get_default_store",
]
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, Text
from sqlalchemy import String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class AIWritingSessionRow(Base):
@@ -22,7 +23,16 @@ class AIWritingSessionRow(Base):
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
# 时间列必须用 BeijingDateTime(项目规范,与 threads_meta 等其它表对齐)—— MySQL
# DATETIME 不存时区,raw ``DateTime(timezone=True)`` 会把 UTC wall clock 写入
# 列,读出来比真实北京时间晚 8 小时。``BeijingDateTime`` 在 bind 时 UTC→Beijing、
# 在 result 时 attach +08:00,前后端拿到的就是北京时间且 tz-aware。
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
@@ -63,6 +63,52 @@ class AIWritingSessionRepository:
await session.refresh(row)
return self._row_to_dict(row)
async def upsert_session(
self,
*,
id: str,
title: str,
user_intent: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
status: str = "in_progress",
) -> dict:
"""幂等:行不存在则创建,行存在则保留原数据(不覆盖 draft / review 等已有写入)。
AI 写作 graph 的 ``intent_parser_node`` 在每次进入时调一次(首启 + resume 时
都会跑),所以必须幂等。resume 时若直接 ``create`` 会因主键冲突抛错;用
``update`` 又找不到行而 no-op。这个方法把两条路合一:行不存在新建,行存在
只刷新 ``updated_at`` 以便列表页排序前置。
"""
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.upsert_session")
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, id)
if row is None:
row = AIWritingSessionRow(
id=id,
user_id=resolved_user_id,
title=title,
user_intent=user_intent,
status=status,
draft_title=None,
draft_markdown=None,
completed_interventions=None,
review_result=None,
transcript=None,
created_at=now,
updated_at=now,
)
session.add(row)
else:
# 行已存在(resume 路径):仅刷新 updated_at,保留 user_id / title / draft / review 等。
# 若 user_id 之前为空(migration 兼容场景),用解析出来的填上。
if row.user_id is None and resolved_user_id is not None:
row.user_id = resolved_user_id
row.updated_at = now
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
session_id: str,
@@ -160,3 +206,38 @@ class AIWritingSessionRepository:
await session.delete(row)
await session.commit()
return True
async def list_older_than(
self,
cutoff: datetime,
*,
only_finished: bool = False,
limit: int = 500,
) -> list[str]:
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
Args:
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
only_finished: True 时只清 status in (done/error),False 不看状态
limit: 单批最多返回多少条,用于分批处理避免大事务
"""
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
if only_finished:
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [row[0] for row in result.all()]
async def delete_by_ids(self, session_ids: list[str]) -> int:
"""按 id 列表批量删除。返回实际删除的行数。"""
if not session_ids:
return 0
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
async with self._sf() as session:
result = await session.execute(stmt)
await session.commit()
return int(result.rowcount or 0)
@@ -10,12 +10,16 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
@@ -26,36 +30,109 @@ def _json_serializer(obj: object) -> str:
logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _add_runtime_vendor_path() -> None:
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
The offline Docker image keeps the Python environment inside the image, while
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
unchanged startup script can still import it after container recreation.
"""
try:
from deerflow.config.runtime_paths import runtime_home
vendor_dir = runtime_home() / "python-vendor"
except Exception:
vendor_dir = Path(".deer-flow/python-vendor").resolve()
vendor = str(vendor_dir)
if vendor_dir.exists() and vendor not in sys.path:
sys.path.insert(0, vendor)
def _mysql_driver_name(url: str) -> str:
if url.startswith("mysql+aiomysql://"):
return "aiomysql"
return "asyncmy"
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _add_runtime_vendor_path() -> None:
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
The offline Docker image keeps the Python environment inside the image, while
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
unchanged startup script can still import it after container recreation.
"""
try:
from deerflow.config.runtime_paths import runtime_home
vendor_dir = runtime_home() / "python-vendor"
except Exception:
vendor_dir = Path(".deer-flow/python-vendor").resolve()
vendor = str(vendor_dir)
if vendor_dir.exists() and vendor not in sys.path:
sys.path.insert(0, vendor)
def _mysql_driver_name(url: str) -> str:
if url.startswith("mysql+aiomysql://"):
return "aiomysql"
return "asyncmy"
async def _run_alembic_upgrade(url: str) -> None:
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
Runs after ``Base.metadata.create_all`` so brand-new deployments still get
a complete schema even without Alembic, while existing deployments pick up
in-place schema fixes (column widenings, new indexes, etc.) at the next
restart — no manual ``alembic upgrade`` step required.
Notes:
* Each historical migration in ``migrations/versions/`` already guards
itself with ``_has_table`` / ``_has_column``, so re-running them against
a database that was originally built via ``create_all`` is a sequence of
no-ops up to the first migration that actually has work to do.
* The Alembic ``env.py`` calls ``asyncio.run`` internally, which cannot run
inside an already-running loop — so we hop into a worker thread.
* Failures here are logged but do not abort startup. The app remains
operable on the ``create_all`` schema; the operator just won't get
in-place fixes until they investigate.
"""
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in ("1", "true", "yes"):
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations (schema stays at create_all)")
return
ini_path = Path(__file__).parent / "migrations" / "alembic.ini"
if not ini_path.exists():
logger.warning("Alembic config not found at %s; skipping auto-migrate", ini_path)
return
# Alembic's ``Config`` is backed by ConfigParser, which performs ``%``
# interpolation on every option value. URL-encoded credentials such as
# ``kland%4088124932`` (the ``@`` in ``kland@88124932``) trip that up with
# ``ValueError: invalid interpolation syntax ... at position N``. Doubling
# the percent signs neutralises the interpolation; ConfigParser un-doubles
# them back to the original value when read.
escaped_url = url.replace("%", "%%")
def _do_upgrade() -> None:
# Import inside the worker so a missing Alembic install can't break
# the engine import chain.
from alembic import command
from alembic.config import Config
cfg = Config(str(ini_path))
cfg.set_main_option("sqlalchemy.url", escaped_url)
# The ``script_location`` in alembic.ini is ``%(here)s`` which points
# at the directory of alembic.ini itself — already correct for us.
command.upgrade(cfg, "head")
# Bound the upgrade so a stuck DDL (e.g. MySQL metadata-lock wait) cannot
# hang startup forever. Tunable via DEER_FLOW_ALEMBIC_TIMEOUT (seconds).
# NOTE: asyncio.wait_for cancels the *await* on timeout; the worker thread
# itself keeps running (Python can't kill threads). That's acceptable —
# the orphan thread will exit when its MySQL connection eventually errors
# or when the process restarts. The point here is to unblock startup.
try:
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
except ValueError:
timeout_s = 60.0
try:
await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s)
logger.info("Alembic migrations applied (or already up-to-date)")
except asyncio.TimeoutError:
logger.error(
"Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. "
"Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.",
timeout_s,
)
except Exception:
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
@@ -79,11 +156,50 @@ async def _auto_create_postgres_db(url: str) -> None:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
finally:
await maint_engine.dispose()
async def init_engine(
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
backend: str,
*,
url: str = "",
@@ -94,8 +210,8 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
echo: Echo SQL to log.
pool_size: Postgres connection pool size.
sqlite_dir: Directory to create for SQLite (ensured to exist).
@@ -106,20 +222,20 @@ async def init_engine(
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
if find_spec(driver) is None:
raise ImportError(
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
if find_spec(driver) is None:
raise ImportError(
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if backend == "sqlite":
import os
@@ -150,29 +266,31 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
# Auto-create tables (dev convenience). Production should use Alembic.
# Auto-create tables for fresh deployments, then let Alembic apply any
# in-place schema fixes on top (see ``_run_alembic_upgrade`` below). Both
# are idempotent so repeating them across restarts is harmless.
from deerflow.persistence.base import Base
# Import all models so Base.metadata discovers them.
@@ -184,21 +302,41 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s", backend)
@@ -209,10 +347,10 @@ async def init_engine_from_config(config) -> None:
await init_engine("memory")
return
await init_engine(
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
@@ -0,0 +1,23 @@
"""Per-user memory configuration override persistence."""
from __future__ import annotations
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
from deerflow.persistence.memory_config.memory import MemoryUserMemoryConfigStore
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.memory_config.sql import SqlUserMemoryConfigRepository
def make_user_memory_config_store(session_factory) -> UserMemoryConfigStore:
if session_factory is None:
return MemoryUserMemoryConfigStore()
return SqlUserMemoryConfigRepository(session_factory)
__all__ = [
"UserMemoryConfigStore",
"UserMemoryConfigRow",
"MemoryUserMemoryConfigStore",
"SqlUserMemoryConfigRepository",
"make_user_memory_config_store",
]
@@ -0,0 +1,32 @@
"""Abstract persistence API for per-user memory config overrides."""
from __future__ import annotations
from abc import ABC, abstractmethod
class UserMemoryConfigStore(ABC):
@abstractmethod
async def get_recall_budget(self, user_id: str) -> str | None:
"""Return the user's recall_budget override, or None if not set."""
raise NotImplementedError
@abstractmethod
async def set_recall_budget(self, user_id: str, value: str) -> None:
"""Upsert the user's recall_budget override."""
raise NotImplementedError
@abstractmethod
async def delete(self, user_id: str) -> None:
"""Remove all per-user overrides for the given user."""
raise NotImplementedError
@abstractmethod
async def load_all(self) -> dict[str, str]:
"""Bulk-load all user recall_budget overrides.
Returns {user_id: recall_budget} for all rows where recall_budget IS NOT NULL.
Called once at startup to warm the in-memory cache.
"""
raise NotImplementedError
@@ -0,0 +1,23 @@
"""In-memory user memory config store (used when database.backend=memory)."""
from __future__ import annotations
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
class MemoryUserMemoryConfigStore(UserMemoryConfigStore):
def __init__(self) -> None:
self._data: dict[str, str] = {}
async def get_recall_budget(self, user_id: str) -> str | None:
return self._data.get(user_id)
async def set_recall_budget(self, user_id: str, value: str) -> None:
self._data[user_id] = value
async def delete(self, user_id: str) -> None:
self._data.pop(user_id, None)
async def load_all(self) -> dict[str, str]:
return dict(self._data)
@@ -0,0 +1,29 @@
"""ORM model for per-user memory configuration overrides."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class UserMemoryConfigRow(Base):
__tablename__ = "user_memory_configs"
# One row per user; upserted on write.
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
# The only user-writable field. NULL means "use global default from config.yaml".
# Allowed values: "low" | "mid" | "high".
recall_budget: Mapped[str | None] = mapped_column(String(16), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,50 @@
"""SQLAlchemy-backed user memory config repository."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
class SqlUserMemoryConfigRepository(UserMemoryConfigStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_recall_budget(self, user_id: str) -> str | None:
async with self._sf() as session:
row = await session.get(UserMemoryConfigRow, user_id)
return row.recall_budget if row is not None else None
async def set_recall_budget(self, user_id: str, value: str) -> None:
async with self._sf() as session:
row = await session.get(UserMemoryConfigRow, user_id)
if row is None:
row = UserMemoryConfigRow(
user_id=user_id,
recall_budget=value,
updated_at=datetime.now(UTC),
)
session.add(row)
else:
row.recall_budget = value
row.updated_at = datetime.now(UTC)
await session.commit()
async def delete(self, user_id: str) -> None:
async with self._sf() as session:
row = await session.get(UserMemoryConfigRow, user_id)
if row is not None:
await session.delete(row)
await session.commit()
async def load_all(self) -> dict[str, str]:
stmt = select(UserMemoryConfigRow).where(UserMemoryConfigRow.recall_budget.is_not(None))
async with self._sf() as session:
result = await session.execute(stmt)
return {row.user_id: row.recall_budget for row in result.scalars() if row.recall_budget is not None}
@@ -9,9 +9,11 @@ from __future__ import annotations
import asyncio
import logging
import os
from logging.config import fileConfig
from alembic import context
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from deerflow.persistence.base import Base
@@ -44,13 +46,27 @@ def run_migrations_offline() -> None:
context.run_migrations()
def do_run_migrations(connection):
render_as_batch = connection.dialect.name == "sqlite"
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
)
def do_run_migrations(connection):
render_as_batch = connection.dialect.name == "sqlite"
# On MySQL, shrink the session lock waits so a blocked DDL fails fast
# instead of hanging on the server default (lock_wait_timeout = 31536000s
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
# transaction on another connection can freeze startup indefinitely.
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
if connection.dialect.name == "mysql":
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
lock_wait = 15
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
)
with context.begin_transaction():
context.run_migrations()
@@ -0,0 +1,64 @@
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
The ``transcript`` column stores the full AI-writing conversation timeline as a
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
a transcript with many progress events overflows it and the save endpoint
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
Revision ID: 20260519_02
Revises: 20260519_01
Create Date: 2026-05-19
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260519_02"
down_revision = "20260519_01"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
bind = op.get_bind()
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=LONGTEXT(),
existing_nullable=True,
)
def downgrade() -> None:
bind = op.get_bind()
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
# is a developer-only path.
import sqlalchemy as sa
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=sa.Text(),
existing_nullable=True,
)
@@ -0,0 +1,83 @@
"""Homepage agent selector: ``user_agent_pin`` table + ``agents.featured_order``.
Adds two pieces backing the new chat-homepage agent picker:
- ``agents.featured_order`` (nullable INT) — admin-curated featured rank.
NULL = not shown on the selector. Non-NULL rows sort ascending.
- ``user_agent_pin`` — per-user pinned agents (user's "favorites"). Composite
PK ``(user_id, agent_id)`` so a user pins each agent at most once. Foreign
key to ``agents.id`` with ``ON DELETE CASCADE`` so deleting an agent also
removes every user's pin of it.
Revision ID: 20260520_01
Revises: 20260519_02
Create Date: 2026-05-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260520_01"
down_revision = "20260519_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
# 1. agents.featured_order — ALTER existing table. Skipped on brand-new DBs
# where create_all already materialised the column.
if _has_table("agents") and not _has_column("agents", "featured_order"):
op.add_column("agents", sa.Column("featured_order", sa.Integer(), nullable=True))
if _has_table("agents") and not _has_index("agents", "ix_agents_featured_order"):
op.create_index("ix_agents_featured_order", "agents", ["featured_order"])
# 2. user_agent_pin — per-user pinned agents.
if not _has_table("user_agent_pin"):
op.create_table(
"user_agent_pin",
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column(
"agent_id",
sa.String(64),
sa.ForeignKey("agents.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("user_id", "agent_id", name="pk_user_agent_pin"),
)
op.create_index("ix_user_agent_pin_sort_order", "user_agent_pin", ["sort_order"])
def downgrade() -> None:
if _has_index("user_agent_pin", "ix_user_agent_pin_sort_order"):
op.drop_index("ix_user_agent_pin_sort_order", table_name="user_agent_pin")
if _has_table("user_agent_pin"):
op.drop_table("user_agent_pin")
if _has_index("agents", "ix_agents_featured_order"):
op.drop_index("ix_agents_featured_order", table_name="agents")
if _has_column("agents", "featured_order"):
op.drop_column("agents", "featured_order")
@@ -0,0 +1,66 @@
"""Widen ``agents.featured_order`` from INT to BIGINT.
The previous revision (``20260520_01``) created ``featured_order`` as plain
``INTEGER``. The frontend seeds the initial rank with ``Date.now()`` so admins
get "newest featured sinks to the bottom" with zero UI — but ms-resolution
timestamps (~1.78e12) overflow a 4-byte signed INT on MySQL, surfacing as
sqlalchemy.exc.DataError: (1264, "Out of range value for column 'featured_order'")
SQLite and PostgreSQL ``INTEGER`` is already 64-bit / arbitrary-precision and
needs no change, but the migration runs everywhere for schema parity.
Revision ID: 20260520_02
Revises: 20260520_01
Create Date: 2026-05-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260520_02"
down_revision = "20260520_01"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("agents", "featured_order"):
return
bind = op.get_bind()
# SQLite has only INTEGER (dynamic precision) — alter_column would rebuild
# the table for nothing. PostgreSQL INTEGER is 32-bit too, so alter it.
if bind.dialect.name == "sqlite":
return
op.alter_column(
"agents",
"featured_order",
type_=sa.BigInteger(),
existing_nullable=True,
)
def downgrade() -> None:
if not _has_column("agents", "featured_order"):
return
bind = op.get_bind()
if bind.dialect.name == "sqlite":
return
# Any rows with values > 2^31-1 would be truncated. Downgrade is dev-only.
op.alter_column(
"agents",
"featured_order",
type_=sa.Integer(),
existing_nullable=True,
)
@@ -0,0 +1,61 @@
"""Drop ``user_agent_pin`` (homepage selector simplified to admin-only).
The original design split the homepage agent picker into two buckets:
admin-curated "featured" + per-user "pinned". Product feedback showed the
per-user side added complexity without clear value — admins now own the
single list shown to everyone. ``agents.featured_order`` is the only piece
that survives; this revision removes the now-unused pin table.
Revision ID: 20260520_03
Revises: 20260520_02
Create Date: 2026-05-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260520_03"
down_revision = "20260520_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_index("user_agent_pin", "ix_user_agent_pin_sort_order"):
op.drop_index("ix_user_agent_pin_sort_order", table_name="user_agent_pin")
if _has_table("user_agent_pin"):
op.drop_table("user_agent_pin")
def downgrade() -> None:
if not _has_table("user_agent_pin"):
op.create_table(
"user_agent_pin",
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column(
"agent_id",
sa.String(64),
sa.ForeignKey("agents.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("user_id", "agent_id", name="pk_user_agent_pin"),
)
op.create_index("ix_user_agent_pin_sort_order", "user_agent_pin", ["sort_order"])
@@ -0,0 +1,44 @@
"""Create user_memory_configs table.
Replaces the per-user file-based memory config override
({base_dir}/users/{user_id}/memory_config.json) with a DB row.
Stores only hindsight.recall_budget — the single user-writable field
after the memory config simplification.
Revision ID: 20260521_01
Revises: 20260519_02
Create Date: 2026-05-21
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260521_01"
down_revision = "20260519_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("user_memory_configs"):
return
op.create_table(
"user_memory_configs",
sa.Column("user_id", sa.String(64), primary_key=True),
sa.Column("recall_budget", sa.String(16), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if _has_table("user_memory_configs"):
op.drop_table("user_memory_configs")
@@ -0,0 +1,24 @@
"""Merge agent pin/featured and user memory config migration heads.
Revision ID: 20260521_02
Revises: 20260520_03, 20260521_01
Create Date: 2026-05-21
"""
from __future__ import annotations
from collections.abc import Sequence
revision: str = "20260521_02"
down_revision: str | tuple[str, str] = ("20260520_03", "20260521_01")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,36 @@
"""Add description column to scheduled_tasks.
Revision ID: 20260522_01
Revises: 20260521_02
Create Date: 2026-05-22
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260522_01"
down_revision = "20260521_02"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_tasks", "description"):
op.add_column(
"scheduled_tasks",
sa.Column("description", sa.Text(), nullable=True),
)
def downgrade() -> None:
if _has_column("scheduled_tasks", "description"):
op.drop_column("scheduled_tasks", "description")
@@ -16,6 +16,7 @@ there is no matching entity directory.
"""
from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
@@ -55,5 +56,6 @@ __all__ = [
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UserMemoryConfigRow",
"UserRow",
]
@@ -31,6 +31,7 @@ class ScheduledTaskRow(Base):
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
name: Mapped[str] = mapped_column(String(127))
description: Mapped[str | None] = mapped_column(Text)
prompt: Mapped[str] = mapped_column(Text)
schedule_text: Mapped[str] = mapped_column(String(256))
cron_expr: Mapped[str] = mapped_column(String(128))
@@ -76,6 +76,7 @@ class ScheduledTaskRepository(ScheduledTaskStore):
source_agent_name=data.get("source_agent_name"),
execution_agent_name=data.get("execution_agent_name"),
name=data["name"],
description=data.get("description"),
prompt=data["prompt"],
schedule_text=data["schedule_text"],
cron_expr=data["cron_expr"],
@@ -56,6 +56,17 @@ class ThreadMetaStore(abc.ABC):
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
pass
@abc.abstractmethod
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
"""Backfill ``assistant_id`` for a thread row.
Used by ``start_run`` when the row was pre-created via ``POST /threads``
without an assistant_id (the SDK's ``client.threads.create()`` doesn't
pass it), and the first run on the thread reveals the assistant. No-op
if the thread does not exist or the owner check fails.
"""
pass
@abc.abstractmethod
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
"""Merge ``metadata`` into the thread's metadata field.
@@ -117,6 +117,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
if record is None:
return
record["assistant_id"] = assistant_id
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
if record is None:
@@ -174,6 +174,20 @@ class ThreadMetaRepository(ThreadMetaStore):
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
await session.commit()
async def update_assistant_id(
self,
thread_id: str,
assistant_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> None:
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_assistant_id")
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
await session.commit()
async def update_metadata(
self,
thread_id: str,
@@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.types import TypeDecorator
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
@@ -39,6 +40,25 @@ class PortableJSON(TypeDecorator):
return json.loads(value)
class PortableLongText(TypeDecorator):
"""A large text column.
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
already unbounded.
"""
impl = Text
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == "mysql":
return dialect.type_descriptor(LONGTEXT())
return dialect.type_descriptor(Text())
class BeijingDateTime(TypeDecorator):
"""Store timestamps as Beijing (UTC+8) wall-clock time.
@@ -33,6 +33,13 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Flush the buffer at least this often (seconds) even when it never fills up.
# Live consumers — notably the scheduled-task run-detail page, which polls
# ``GET /runs/{rid}/messages`` every 4s — would otherwise see nothing until a
# run ends, because a short run can finish before the buffer reaches
# ``flush_threshold``.
_FLUSH_INTERVAL_SECONDS = 3.0
class RunJournal(BaseCallbackHandler):
"""LangChain callback handler that captures events to RunEventStore."""
@@ -56,6 +63,9 @@ class RunJournal(BaseCallbackHandler):
# Write buffer
self._buffer: list[dict] = []
self._pending_flush_tasks: set[asyncio.Task[None]] = set()
# Monotonic timestamp of the last flush attempt — drives time-based
# flushing so incremental progress is visible to live pollers.
self._last_flush_at: float = time.monotonic()
# Token accumulators
self._total_input_tokens = 0
@@ -269,8 +279,14 @@ class RunJournal(BaseCallbackHandler):
"created_at": datetime.now(UTC).isoformat(),
}
)
# Flush on a full buffer, or once enough time has elapsed since the
# last flush. The time-based path lets live consumers (the
# scheduled-task run-detail page polls every 4s) see incremental
# progress even when a run ends before the buffer fills up.
if len(self._buffer) >= self._flush_threshold:
self._flush_sync()
elif time.monotonic() - self._last_flush_at >= _FLUSH_INTERVAL_SECONDS:
self._flush_sync()
def _flush_sync(self) -> None:
"""Best-effort flush of buffer to RunEventStore.
@@ -293,6 +309,7 @@ class RunJournal(BaseCallbackHandler):
return
batch = self._buffer.copy()
self._buffer.clear()
self._last_flush_at = time.monotonic()
task = loop.create_task(self._flush_async(batch))
self._pending_flush_tasks.add(task)
task.add_done_callback(self._on_flush_done)
@@ -307,8 +324,10 @@ class RunJournal(BaseCallbackHandler):
self.run_id,
exc_info=True,
)
# Return failed events to buffer for retry on next flush
# Return failed events to buffer for retry on next flush, then
# re-raise so _on_flush_done can tell success from failure.
self._buffer = batch + self._buffer
raise
def _on_flush_done(self, task: asyncio.Task) -> None:
self._pending_flush_tasks.discard(task)
@@ -316,7 +335,16 @@ class RunJournal(BaseCallbackHandler):
return
exc = task.exception()
if exc:
# The batch was already returned to the buffer by _flush_async.
# Don't retry here — that would busy-loop on a persistent DB
# outage; the next _put (or the final flush()) retries instead.
logger.warning("Journal flush task failed: %s", exc)
return
# Events may have accumulated while this flush was in flight — drain
# them now so incremental progress keeps reaching the store without
# waiting for the next _put to cross a threshold.
if self._buffer:
self._flush_sync()
def _identify_caller(self, tags: list[str] | None) -> str:
_tags = tags or []
@@ -130,8 +130,16 @@ async def run_agent(
stream_subgraphs: bool = False,
interrupt_before: list[str] | Literal["*"] | None = None,
interrupt_after: list[str] | Literal["*"] | None = None,
command: dict[str, Any] | None = None,
) -> None:
"""Execute an agent in the background, publishing events to *bridge*."""
"""Execute an agent in the background, publishing events to *bridge*.
``command`` 来自 LangGraph SDK 的 ``Command`` 协议(``{resume, update, goto}``)。
存在时把它翻译成 ``langgraph.types.Command`` 并作为 ``astream`` 的入参 ——
Pregel 检查到 Command 输入会从最近 checkpoint 处恢复(典型场景:interrupt 之后
的 ``Command(resume=user_input)``);不存在时仍按 ``graph_input`` dict 起新 run。
AI 写作的 4 个干预卡就靠这个把 user_input 传回 ``pause_*_confirm`` 节点。
"""
# Unpack infrastructure dependencies from RunContext.
checkpointer = ctx.checkpointer
@@ -269,10 +277,23 @@ async def run_agent(
logger.info("Run %s: streaming with modes %s (requested: %s)", run_id, lg_modes, requested_modes)
# 7. Stream using graph.astream
# 如果客户端传了 Command(典型:interrupt 后续提的 ``Command(resume=...)``),
# 把它翻译成 langgraph.types.Command 作为 astream 输入;否则按 graph_input dict
# 起新一轮。Pregel 检查到 Command 会从最近 checkpoint 处恢复,而不是重头跑。
stream_input: Any = graph_input
if command is not None:
from langgraph.types import Command as _LGCommand
stream_input = _LGCommand(
resume=command.get("resume"),
update=command.get("update"),
goto=command.get("goto"),
)
if len(lg_modes) == 1 and not stream_subgraphs:
# Single mode, no subgraphs: astream yields raw chunks
single_mode = lg_modes[0]
async for chunk in agent.astream(graph_input, config=runnable_config, stream_mode=single_mode):
async for chunk in agent.astream(stream_input, config=runnable_config, stream_mode=single_mode):
if record.abort_event.is_set():
logger.info("Run %s abort requested — stopping", run_id)
break
@@ -281,7 +302,7 @@ async def run_agent(
else:
# Multiple modes or subgraphs: astream yields tuples
async for item in agent.astream(
graph_input,
stream_input,
config=runnable_config,
stream_mode=lg_modes,
subgraphs=stream_subgraphs,
@@ -18,6 +18,7 @@ from zoneinfo import ZoneInfo
from croniter import croniter
from langgraph.checkpoint.base import empty_checkpoint
from deerflow.config.agents_config import AGENT_ID_PATTERN
from deerflow.config.paths import get_paths
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
from deerflow.runtime import serialize_channel_values
@@ -305,12 +306,51 @@ class ScheduledTaskService:
"""
run = await self.store.get_run(run_id, user_id)
if run is None:
# Either the run doesn't exist, or the caller is a subscriber, not owner.
candidate = await self.store.get_run_any(run_id)
if candidate is None:
raise FileNotFoundError(f"Run {run_id} not found")
raise PermissionError("Only the run owner may edit result files")
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
async def admin_update_run_file(
self,
run_id: str,
file_index: int,
new_content: str,
) -> dict[str, Any]:
"""Owner-less variant of :meth:`update_run_file` used by the public viewer.
The public scheduled-task page is unauthenticated, so it cannot supply a
``user_id``. Mutations from that page route here instead. The disk-write
and result-row update logic is identical to the owner path.
"""
run = await self.store.get_run_any(run_id)
if run is None:
raise FileNotFoundError(f"Run {run_id} not found")
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
async def admin_update_run_body(self, run_id: str, new_content: str) -> dict[str, Any]:
"""Overwrite the inline Markdown body (``result.content``) of a run.
Used by the public scheduled-task viewer so anonymous editors can fix
typos in the "正文" tab. Unlike :meth:`admin_update_run_file`, no disk
artifact backs the body — it lives entirely in the run row.
"""
run = await self.store.get_run_any(run_id)
if run is None:
raise FileNotFoundError(f"Run {run_id} not found")
result = run.get("result") if isinstance(run.get("result"), dict) else {}
new_result = {**result, "content": new_content}
await self.store.update_run(run_id, {"result": new_result})
return {"content": new_content}
async def _apply_run_file_edit(
self,
run: dict[str, Any],
run_id: str,
file_index: int,
new_content: str,
) -> dict[str, Any]:
result = run.get("result") or {}
files = result.get("files") if isinstance(result, dict) else None
if not isinstance(files, list) or file_index < 0 or file_index >= len(files):
@@ -406,6 +446,7 @@ class ScheduledTaskService:
prompt: str,
schedule: str,
timezone: str | None = None,
description: str | None = None,
source_thread_id: str | None = None,
source_agent_name: str | None = None,
execution_agent_name: str | None = None,
@@ -420,6 +461,7 @@ class ScheduledTaskService:
context = dict(execution_context or {})
if execution_agent_name:
context["agent_name"] = execution_agent_name
normalized_description = description.strip() if isinstance(description, str) else None
task = await self.store.create_task(
{
"task_id": str(uuid.uuid4()),
@@ -429,6 +471,7 @@ class ScheduledTaskService:
"source_agent_name": source_agent_name,
"execution_agent_name": execution_agent_name,
"name": final_name,
"description": normalized_description or None,
"prompt": prompt.strip(),
"schedule_text": schedule.strip(),
"cron_expr": cron_expr,
@@ -451,6 +494,15 @@ class ScheduledTaskService:
for key in ("name", "prompt", "enabled", "execution_agent_name"):
if key in data and data[key] is not None:
update_data[key] = data[key]
# Description is nullable — accept ``None`` (clear) and empty strings
# (also clear) explicitly, so the field can be removed via PATCH.
if "description" in data:
raw = data["description"]
if isinstance(raw, str):
trimmed = raw.strip()
update_data["description"] = trimmed or None
else:
update_data["description"] = None
# Names are matched against the per-user uniqueness constraint, so keep
# the stored value trimmed to match create_task's normalization.
if isinstance(update_data.get("name"), str):
@@ -661,7 +713,7 @@ class ScheduledTaskService:
retry_count: int = 0,
) -> None:
"""Mark a run failed and reflect the outcome on the parent task."""
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
result = _build_result(task, scheduled_for=scheduled_for, content=f"定时任务执行失败:{error}")
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
@@ -819,8 +871,17 @@ class ScheduledTaskService:
try:
request = SimpleNamespace(app=self.app)
execution_context = dict(task.get("execution_context") or {})
if task.get("execution_agent_name"):
execution_context["agent_name"] = task["execution_agent_name"]
stored_agent_ref = task.get("execution_agent_name")
if stored_agent_ref:
resolved_agent_id = await self._resolve_agent_id(stored_agent_ref, task["user_id"])
if resolved_agent_id:
execution_context["agent_name"] = resolved_agent_id
else:
logger.warning(
"Scheduled task %s references unknown agent %r; falling back to default lead agent",
task.get("task_id"),
stored_agent_ref,
)
execution_context.update(
{
"is_scheduled_run": True,
@@ -845,6 +906,35 @@ class ScheduledTaskService:
finally:
reset_current_user(token)
async def _resolve_agent_id(self, stored_ref: str, user_id: str) -> str | None:
"""Resolve a scheduled task's ``execution_agent_name`` to a stable agent id.
Older tasks (created before the frontend was fixed) stored the agent's
Chinese **display name** here; the runtime validator only accepts ids
matching ``^[A-Za-z0-9_-]+$``, so those rows would 422 forever. We
gracefully recover by looking the stored value up first as an id, then
as a display name. Returns ``None`` when nothing visible matches — the
caller then falls back to the default lead agent rather than crashing.
"""
store = getattr(self.app.state, "agent_store", None)
if store is None:
return stored_ref if AGENT_ID_PATTERN.fullmatch(stored_ref) else None
if AGENT_ID_PATTERN.fullmatch(stored_ref):
try:
row = await store.get_visible(stored_ref, user_id)
except Exception:
row = None
if row is not None:
return row["id"]
try:
agents = await store.list_visible(user_id)
except Exception:
return None
for agent in agents:
if agent.get("name") == stored_ref:
return agent.get("id")
return None
async def _extract_latest_ai_text(self, thread_id: str) -> str | None:
checkpointer = self.app.state.checkpointer
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
@@ -10,6 +10,7 @@ Consumers: ``deerflow.runtime.runs.worker`` (SSE publishing) and
from __future__ import annotations
import dataclasses
from typing import Any
@@ -35,6 +36,16 @@ def serialize_lc_object(obj: Any) -> Any:
return obj.dict()
except Exception:
pass
# Dataclass instances —— LangGraph 的 ``Interrupt`` 就是 dataclass 没有
# model_dump 也没有 dict 方法;之前 fallback 到 ``str(obj)`` 会把
# ``Interrupt(value=..., id='...')`` 序列化成字符串,下游 SDK
# (``@langchain/langgraph-sdk``) 期待 dict 结构 → 直接丢弃 → AI 写作
# 的 4 个干预卡永远不弹(参 ai-writing-优化.md §阶段 3 排查记录)。
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
try:
return {k: serialize_lc_object(v) for k, v in dataclasses.asdict(obj).items()}
except Exception:
pass
# Last resort
try:
return str(obj)
@@ -45,12 +56,21 @@ def serialize_lc_object(obj: Any) -> Any:
def serialize_channel_values(channel_values: dict[str, Any]) -> dict[str, Any]:
"""Serialize channel values, stripping internal LangGraph keys.
Internal keys like ``__pregel_*`` and ``__interrupt__`` are removed
to match what the LangGraph Platform API returns.
``__pregel_*`` 键始终剥离。``__interrupt__`` 仅在**为空时**剥离 —— 因为
``@langchain/langgraph-sdk`` 的 ``useStream`` 只在 ``event === "values"``
通道里读 ``data.__interrupt__`` 并通过 ``stream.interrupt`` 暴露给上层。
AI 写作的 4 个干预卡(material/outline/draft/review)就是靠它触发的;
继续过滤会让前端永远拿不到 interrupt。
主聊天不消费 ``stream.interrupt``,但若我们让一个空数组溜过去,SDK 会
fallback 成 ``{when: "breakpoint"}`` 的伪 interrupt —— 所以仅在数组非空
时保留,两侧都不破坏。
"""
result: dict[str, Any] = {}
for key, value in channel_values.items():
if key.startswith("__pregel_") or key == "__interrupt__":
if key.startswith("__pregel_"):
continue
if key == "__interrupt__" and not value:
continue
result[key] = serialize_lc_object(value)
return result
@@ -39,6 +39,12 @@ def _extract_json_object(raw: str) -> dict | None:
async def scan_skill_content(content: str, *, executable: bool = False, location: str = SKILL_MD_FILE, app_config: AppConfig | None = None) -> ScanResult:
"""Screen skill content before it is written to disk."""
config = app_config or get_app_config()
# Operator-level kill switch: when ``skills.security_scan.enabled`` is false,
# bypass the LLM round-trip entirely and let the content through.
if not config.skills.security_scan.enabled:
return ScanResult("allow", "Security scan disabled by config.")
rubric = (
"You are a security reviewer for AI agent skills. "
"Classify the content as allow, warn, or block. "
@@ -49,7 +55,6 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
prompt = f"Location: {location}\nExecutable: {str(executable).lower()}\n\nReview this content:\n-----\n{content}\n-----"
try:
config = app_config or get_app_config()
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
model_name = config.skill_evolution.curator.model_name
@@ -1,3 +1,4 @@
from .agent_orchestration_tool import agent_orchestration_tool
from .clarification_tool import ask_clarification_tool
from .memory_tool import memory_tool
from .present_file_tool import present_file_tool
@@ -10,6 +11,7 @@ __all__ = [
"setup_agent",
"present_file_tool",
"ask_clarification_tool",
"agent_orchestration_tool",
"memory_tool",
"scheduled_task_tool",
"view_image_tool",
@@ -0,0 +1,46 @@
"""Placeholder tool that lets the roundtable leader agent dispatch sub-agents.
Background
----------
The multi-agent ``/api/multi-agent/run/stream`` endpoint (``app.gateway.routers.multi_agent``)
drives a *leader* agent whose only job is to delegate work to seat-specific sub-agents.
Its SOUL.md tells the model to "use the agent_orchestration skill, passing agent_name
and task". With no real tool registered under that name, the model used to emit
``{"name": "agent_orchestration", "args": {"agent_name": ..., "task": ...}}`` calls
that the runtime rejected with "agent_orchestration is not a valid tool", driving it
into an unproductive retry loop (a single leader call could exceed 10 000 SSE frames).
This module exposes a no-op tool with the exact signature the leader naturally
emits. The actual dispatch is handled by ``SkillStopMiddleware`` which intercepts
the call, halts the run, and lets ``multi_agent.py`` parse the AIMessage's
``tool_calls`` to extract ``(agent_name, task)`` pairs for the frontend.
"""
from langchain.tools import tool
@tool("agent_orchestration", parse_docstring=True)
def agent_orchestration_tool(agent_name: str, task: str) -> str:
"""Delegate a task to a named sub-agent in the multi-agent roundtable.
Use this tool ONLY when you are the roundtable coordinator and need to
dispatch work to one of the configured seat agents. Each call delegates
ONE task to ONE agent. Emit multiple calls in the same turn to dispatch
several seats in a single round of the discussion.
The call is intercepted by the runtime — you do not get a real result
back synchronously. The orchestrator will collect every dispatched
``(agent_name, task)`` pair, run the corresponding sub-agents, then
re-prompt you with a synthesis instruction once they finish.
Args:
agent_name: Name of the sub-agent to dispatch. MUST exactly match one of
the names listed in your SOUL prompt's ``可调度智能体`` section.
task: The instruction to hand to that sub-agent. Be specific — the
sub-agent only sees this string (plus its own SOUL prompt), not the
broader conversation context.
"""
# Real handling is in SkillStopMiddleware + the multi_agent router. This
# body only runs as a fallback if the middleware is not configured; it
# returns a placeholder so the model does not see a "not a valid tool" error.
return f"dispatched:{agent_name}"
@@ -36,18 +36,31 @@ def ask_clarification_tool(
- You're about to perform a potentially dangerous operation
- You have a recommendation but need user approval
Batching multiple independent questions:
- **You CAN and SHOULD call ask_clarification multiple times in the same turn** when the user's
request leaves several **independent** aspects unclear (e.g. project type AND tech stack AND
deployment target). Emit one tool call per aspect — do NOT cram several unrelated questions
into one `question` string.
- The frontend automatically groups all clarification cards in one turn into a single submission,
so the user answers them together in one reply. You will then receive one combined user message
addressing every question in order.
- When the questions DEPEND on each other (the second only makes sense after the first is answered),
ask only the first one and wait for the reply before asking the next.
Best practices:
- Ask ONE clarification at a time for clarity
- Be specific and clear in your question
- Don't make assumptions when clarification is needed
- For risky operations, ALWAYS ask for confirmation
- After calling this tool, execution will be interrupted automatically
- One question per ask_clarification call — keep each `question` focused on a single decision.
- Be specific and clear in your question.
- Provide `options` (with concise labels) whenever there is a discrete set of reasonable choices —
this lets the user multi-select in the UI instead of typing free text.
- Don't make assumptions when clarification is needed.
- For risky operations, ALWAYS ask for confirmation.
- After calling this tool (or several in one turn), execution will be interrupted automatically.
Args:
question: The clarification question to ask the user. Be specific and clear.
question: The clarification question to ask the user. Focused on a single decision; be specific and clear.
clarification_type: The type of clarification needed (missing_info, ambiguous_requirement, approach_choice, risk_confirmation, suggestion).
context: Optional context explaining why clarification is needed. Helps the user understand the situation.
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from.
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from; the UI supports multi-select.
"""
# This is a placeholder implementation
# The actual logic is handled by ClarificationMiddleware which intercepts this tool call
@@ -82,7 +82,19 @@ def memory_tool(
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
if not config.enabled or not config.builtin.enabled:
if not config.enabled:
return json.dumps(
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
ensure_ascii=False,
)
if getattr(config, "version", "v2") == "v1":
return json.dumps(
{"success": False, "error": "当前使用 V1 记忆模式,记忆由系统自动提取,不支持手动写入。"},
ensure_ascii=False,
)
if not config.builtin.enabled:
return json.dumps(
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
ensure_ascii=False,
@@ -6,7 +6,7 @@ from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.reflection import resolve_variable
from deerflow.sandbox.security import is_host_bash_allowed
from deerflow.tools.builtins import ask_clarification_tool, memory_tool, present_file_tool, scheduled_task_tool, task_tool, view_image_tool
from deerflow.tools.builtins import agent_orchestration_tool, ask_clarification_tool, memory_tool, present_file_tool, scheduled_task_tool, task_tool, view_image_tool
from deerflow.tools.builtins.tool_search import reset_deferred_registry
logger = logging.getLogger(__name__)
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
BUILTIN_TOOLS = [
present_file_tool,
ask_clarification_tool,
agent_orchestration_tool,
]
SUBAGENT_TOOLS = [
@@ -39,6 +40,7 @@ def get_available_tools(
model_name: str | None = None,
subagent_enabled: bool = False,
is_scheduled_run: bool = False,
excluded_tools: list[str] | None = None,
*,
app_config: AppConfig | None = None,
) -> list[BaseTool]:
@@ -52,6 +54,9 @@ def get_available_tools(
include_mcp: Whether to include tools from MCP servers (default: True).
model_name: Optional model name to determine if vision tools should be included.
subagent_enabled: Whether to include subagent tools (task, task_status).
excluded_tools: Optional list of tool names to exclude from the result
(applied AFTER all other resolution). Takes precedence over
``app_config.excluded_tools``.
Returns:
List of available tools.
@@ -209,4 +214,19 @@ def get_available_tools(
"Duplicate tool name %r detected and skipped — check your config.yaml and MCP server registrations (issue #1803).",
t.name,
)
# Drop tools the caller (or config) wants excluded. Request-level
# ``excluded_tools`` wins; the config-level list is the static fallback.
effective_excluded = excluded_tools if excluded_tools else getattr(config, "excluded_tools", None) or []
if effective_excluded:
excluded_set = set(effective_excluded)
filtered_tools = [t for t in unique_tools if t.name not in excluded_set]
if len(filtered_tools) < len(unique_tools):
logger.info(
"Excluded %d tool(s) by request/config: %s",
len(unique_tools) - len(filtered_tools),
effective_excluded,
)
unique_tools = filtered_tools
return unique_tools
+50 -48
View File
@@ -1,51 +1,53 @@
[project]
name = "deer-flow"
version = "0.1.0"
description = "LangGraph-based AI agent system with sandbox execution capabilities"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"deerflow-harness",
"fastapi>=0.115.0",
"httpx>=0.28.0",
"python-multipart>=0.0.26",
"sse-starlette>=2.1.0",
"uvicorn[standard]>=0.34.0",
"lark-oapi>=1.4.0",
"slack-sdk>=3.33.0",
"python-telegram-bot>=21.0",
"langgraph-sdk>=0.1.51",
"markdown-to-mrkdwn>=0.3.1",
"wecom-aibot-python-sdk>=0.1.6",
"dingtalk-stream>=0.24.3",
"bcrypt>=4.0.0",
"pyjwt>=2.9.0",
"email-validator>=2.0.0",
"croniter>=6.2.2",
]
[project]
name = "deer-flow"
version = "0.1.0"
description = "LangGraph-based AI agent system with sandbox execution capabilities"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"deerflow-harness",
"fastapi>=0.115.0",
"httpx>=0.28.0",
"python-multipart>=0.0.26",
"sse-starlette>=2.1.0",
"uvicorn[standard]>=0.34.0",
"lark-oapi>=1.4.0",
"slack-sdk>=3.33.0",
"python-telegram-bot>=21.0",
"langgraph-sdk>=0.1.51",
"markdown-to-mrkdwn>=0.3.1",
"wecom-aibot-python-sdk>=0.1.6",
"dingtalk-stream>=0.24.3",
"bcrypt>=4.0.0",
"pyjwt>=2.9.0",
"email-validator>=2.0.0",
"croniter>=6.2.2",
"hindsight-client>=0.4.22",
"asyncmy>=0.2.11",
]
[project.optional-dependencies]
postgres = ["deerflow-harness[postgres]"]
mysql = ["deerflow-harness[mysql]"]
[dependency-groups]
dev = [
"prompt-toolkit>=3.0.0",
"pytest>=9.0.3",
"pytest-asyncio>=1.3.0",
"ruff>=0.14.11",
]
[tool.pytest.ini_options]
markers = [
"no_auto_user: disable the conftest autouse contextvar fixture for this test",
]
[tool.uv]
index-url = "https://pypi.org/simple"
[tool.uv.workspace]
members = ["packages/harness"]
[tool.uv.sources]
deerflow-harness = { workspace = true }
[dependency-groups]
dev = [
"prompt-toolkit>=3.0.0",
"pytest>=9.0.3",
"pytest-asyncio>=1.3.0",
"ruff>=0.14.11",
]
[tool.pytest.ini_options]
markers = [
"no_auto_user: disable the conftest autouse contextvar fixture for this test",
]
[tool.uv]
index-url = "https://pypi.org/simple"
[tool.uv.workspace]
members = ["packages/harness"]
[tool.uv.sources]
deerflow-harness = { workspace = true }
@@ -0,0 +1,179 @@
"""AI 写作 assistant 冒烟脚本:验证 ai_writing graph 注册到 LangGraph Server 后,
interrupt/Command(resume=...)/state 持久化/custom stream 全链路是否跑通。
阶段 0 用名为 ai_writing_probe;阶段 1 起改为正式 assistant 名 ai_writing,本脚本随之切换。
前置条件:
1. langgraph.json 已注册 ai_writing(见 ai-writing-优化.md 阶段 1)。
2. 后端已通过 `make dev` 启动,Gateway + LangGraph Server 监听 8001。
3. config.yaml 配好至少一个可用 model,researcher 用到的 search_provider 也已配置
—— 否则 researcher 节点会失败(这是业务依赖,不是迁移问题,仍然算「跑通」)。
运行:
cd offline-backend-20260512/backend
PYTHONPATH=. uv run python scripts/probe_ai_writing_assistant.py
通过条件(与 ai-writing-优化.md §阶段 0 §通过条件 对齐):
- assistants 列表里能看到 ai_writing_probe
- 跑到 pause_material_confirm 能拿到 interrupt 事件
- Command(resume={"action": "confirm", ...}) 能成功续跑
- 重建客户端后 client.threads.get_state(thread_id) 仍能拿到上一阶段 state
"""
from __future__ import annotations
import asyncio
import os
import sys
from typing import Any
try:
from langgraph_sdk import get_client
except ImportError:
print("ERROR: langgraph-sdk 未安装。run: uv pip install langgraph-sdk", file=sys.stderr)
sys.exit(1)
ASSISTANT_ID = "ai_writing"
DEFAULT_URL = os.environ.get("DEERFLOW_LANGGRAPH_URL", "http://localhost:8001/api/langgraph")
def _banner(text: str) -> None:
print(f"\n{'=' * 8} {text} {'=' * 8}")
def _summarize_event(event: Any) -> str:
"""裁剪日志,避免一屏全是大段 markdown / 素材列表。"""
s = repr(event)
return s if len(s) <= 240 else s[:240] + "…"
async def _stream_until_interrupt(
client, thread_id: str, *, input_payload: dict | None = None, command: dict | None = None
) -> dict | None:
"""跑一次 runs.stream,捕获第一个 interrupt(含 pause payload)后返回,否则返回 None。
收到 interrupt 时把 payload 一并打出来,便于人工核对 4 个 pause 点的 schema。
"""
interrupt_payload: dict | None = None
kwargs: dict[str, Any] = {
"stream_mode": ["values", "messages-tuple", "custom", "updates"],
}
if input_payload is not None:
kwargs["input"] = input_payload
if command is not None:
kwargs["command"] = command
async for chunk in client.runs.stream(thread_id, ASSISTANT_ID, **kwargs):
# langgraph-sdk 把不同 mode 的事件混在同一 stream 里,按 chunk.event 分类。
event_name = getattr(chunk, "event", None) or (
chunk.get("event") if isinstance(chunk, dict) else None
)
data = getattr(chunk, "data", None) or (
chunk.get("data") if isinstance(chunk, dict) else None
)
print(f"[stream] event={event_name} data={_summarize_event(data)}")
# SDK 在遇到 interrupt() 时会发一个特殊事件 —— 不同版本字段名不同,做下兼容。
if event_name == "updates" and isinstance(data, dict):
for _node, node_update in data.items():
if isinstance(node_update, dict) and "__interrupt__" in node_update:
interrupt_payload = node_update["__interrupt__"]
if event_name in {"interrupt", "__interrupt__"}:
interrupt_payload = data
break
return interrupt_payload
async def main() -> int:
print(f"LangGraph URL = {DEFAULT_URL}")
client = get_client(url=DEFAULT_URL)
# ── Step 1: 确认 assistant 已注册 ──────────────────────────────────────────
_banner("Step 1: 列 assistant,确认 probe 已注册")
assistants = await client.assistants.search()
names = [a.get("name") or a.get("graph_id") for a in assistants]
print(f"已注册 assistants: {names}")
if not any(ASSISTANT_ID in (n or "") for n in names):
print(f"FAIL: 未找到 {ASSISTANT_ID},请检查 langgraph.json 注册是否生效。")
return 1
# ── Step 2: 创建 thread 并触发第一次 run ──────────────────────────────────
_banner("Step 2: 创建 thread + 启动写作(应停在 pause_material)")
thread = await client.threads.create()
thread_id = thread["thread_id"]
print(f"thread_id = {thread_id}")
start_input = {
"user_intent": "写一篇关于 LangGraph 多 worker 部署最佳实践的分析报道,目标读者是后端工程师,约 800 字",
"article_type": "analysis",
"target_audience": "后端工程师",
"word_count_target": 800,
"keyword_count": 4,
"writing_mode": "loose",
"max_revisions": 2,
"status": "researching",
"drafts": [],
"progress_events": [],
}
payload = await _stream_until_interrupt(client, thread_id, input_payload=start_input)
if not payload:
print("FAIL: 未收到 interrupt 事件 —— pause_material 可能没停住,或 researcher 抛错。")
return 2
print(f"\nPAUSE-1 payload = {_summarize_event(payload)}")
# ── Step 3: 确认素材,应停在 pause_outline ────────────────────────────────
_banner("Step 3: Command(resume) 确认素材(应停在 pause_outline)")
payload = await _stream_until_interrupt(
client, thread_id,
command={"resume": {"action": "confirm"}}, # 不指定 approvedMaterialIds → 默认全部通过
)
if not payload:
print("FAIL: 未停在 pause_outline。")
return 3
print(f"\nPAUSE-2 payload = {_summarize_event(payload)}")
# ── Step 4: 确认大纲,应停在 pause_draft 或 pause_section_help ───────────
_banner("Step 4: Command(resume) 确认大纲(应停在 pause_draft 或 pause_section_help)")
payload = await _stream_until_interrupt(
client, thread_id,
command={"resume": {"action": "confirm"}},
)
if not payload:
print("FAIL: 未停在 pause_draft / pause_section_help。")
return 4
print(f"\nPAUSE-3 payload = {_summarize_event(payload)}")
# ── Step 5: 定稿 ─────────────────────────────────────────────────────────
_banner("Step 5: Command(resume) finalize 草稿(应跑到 END)")
payload = await _stream_until_interrupt(
client, thread_id,
command={"resume": {"action": "finalize"}},
)
if payload:
print(f"NOTE: 收到额外 interrupt(可能编辑审核打回) payload = {_summarize_event(payload)}")
print("尝试强制定稿…")
payload = await _stream_until_interrupt(
client, thread_id,
command={"resume": {"action": "force_finalize", "overrideVerdict": True}},
)
if payload:
print("WARN: 强制定稿后仍有 interrupt,graph 路由可能有意外分支。")
# ── Step 6: 验证 state 持久化 ────────────────────────────────────────────
_banner("Step 6: 重新建客户端 client.threads.get_state(thread_id) 应能拿到最终 state")
client2 = get_client(url=DEFAULT_URL)
state = await client2.threads.get_state(thread_id)
values = state.get("values") or {}
final_status = values.get("status")
has_draft = bool(values.get("current_draft"))
print(f"final status = {final_status}, has_draft = {has_draft}, "
f"next nodes = {state.get('next')}")
if final_status not in {"done", "writing_draft", "revising"}:
print("WARN: 最终 status 不是 done,请人工确认上面流程是否正常。")
print("\nPROBE OK —— 阶段 0 验证通过的核心条件已成立(interrupt + resume + state 持久化)。")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
@@ -0,0 +1,138 @@
"""Tests for the ``POST /api/skills/reconcile-custom`` adoption flow.
Covers the scenario where a user drops a skill directory onto disk by hand
(bypassing the upload API) and then calls the reconcile route so it shows up
in the UI.
"""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.gateway.routers import skills as skills_router
from deerflow.persistence.skills import MemorySkillStore
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
def _write_skill_on_disk(root: Path, name: str, description: str = "from disk") -> None:
"""Drop a valid SKILL.md into ``<root>/custom/<name>/`` like a manual unzip would."""
skill_dir = root / "custom" / name
skill_dir.mkdir(parents=True, exist_ok=True)
(skill_dir / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: {description}\n---\n# {name}\n",
encoding="utf-8",
)
def _fake_request(user_id: str | None = "user-a") -> SimpleNamespace:
user = SimpleNamespace(id=user_id, system_role="user") if user_id else None
return SimpleNamespace(state=SimpleNamespace(user=user))
@pytest.fixture
def skills_root(tmp_path: Path) -> Path:
root = tmp_path / "skills"
(root / "public").mkdir(parents=True)
(root / "custom").mkdir(parents=True)
return root
@pytest.fixture
def patched_storage(
skills_root: Path, monkeypatch: pytest.MonkeyPatch
) -> LocalSkillStorage:
"""Force the router to use a LocalSkillStorage rooted at ``skills_root``."""
storage = LocalSkillStorage(host_path=str(skills_root))
monkeypatch.setattr(
skills_router, "get_or_new_skill_storage", lambda **_: storage
)
async def _noop_refresh() -> None:
return None
monkeypatch.setattr(
skills_router, "refresh_skills_system_prompt_cache_async", _noop_refresh
)
return storage
@pytest.mark.asyncio
async def test_reconcile_adopts_orphan_custom_skill(
skills_root: Path, patched_storage: LocalSkillStorage
):
_write_skill_on_disk(skills_root, "manually-dropped")
store = MemorySkillStore()
result = await skills_router.reconcile_custom_skills(
request=_fake_request("user-a"), config=None, skill_store=store
)
assert result.adopted == ["manually-dropped"]
assert result.already_registered == []
record = await store.get_any("manually-dropped")
assert record is not None
assert record["owner_user_id"] == "user-a"
assert record["published"] is False
@pytest.mark.asyncio
async def test_reconcile_skips_already_registered(
skills_root: Path, patched_storage: LocalSkillStorage
):
_write_skill_on_disk(skills_root, "already-known")
store = MemorySkillStore()
# Pre-existing ownership row — should be left alone.
await store.create(
{"name": "already-known", "owner_user_id": "user-b", "published": True}
)
result = await skills_router.reconcile_custom_skills(
request=_fake_request("user-a"), config=None, skill_store=store
)
assert result.adopted == []
assert result.already_registered == ["already-known"]
# Ownership must not be overwritten.
record = await store.get_any("already-known")
assert record is not None
assert record["owner_user_id"] == "user-b"
assert record["published"] is True
@pytest.mark.asyncio
async def test_reconcile_mixed_adopts_only_orphans(
skills_root: Path, patched_storage: LocalSkillStorage
):
_write_skill_on_disk(skills_root, "orphan-one")
_write_skill_on_disk(skills_root, "orphan-two")
_write_skill_on_disk(skills_root, "registered")
store = MemorySkillStore()
await store.create(
{"name": "registered", "owner_user_id": "user-a", "published": False}
)
result = await skills_router.reconcile_custom_skills(
request=_fake_request("user-a"), config=None, skill_store=store
)
assert sorted(result.adopted) == ["orphan-one", "orphan-two"]
assert result.already_registered == ["registered"]
@pytest.mark.asyncio
async def test_reconcile_with_empty_custom_dir_is_noop(
skills_root: Path, patched_storage: LocalSkillStorage
):
store = MemorySkillStore()
result = await skills_router.reconcile_custom_skills(
request=_fake_request("user-a"), config=None, skill_store=store
)
assert result.adopted == []
assert result.already_registered == []
+2706 -2699
View File
File diff suppressed because it is too large. Load diff
+39 -2
View File
@@ -37,6 +37,26 @@ models:
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: minimax-m2.7
display_name: MiniMax M2.7
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.7
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: minimax-m2.5
display_name: MiniMax M2.5
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.5
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
tool_groups:
- name: web
- name: file:read
@@ -89,6 +109,8 @@ title:
Assistant: {assistant_msg}
skills:
path: /app/skills
security_scan:
enabled: false
agents_api:
enabled: true
database:
@@ -97,13 +119,28 @@ database:
pool_size: 10
echo_sql: false
checkpointer:
# SQLite 文件路径用相对路径,由 resolve_sqlite_conn_str 解析成绝对路径。
# Docker 部署:把容器内 .deer-flow 目录挂卷出来即可持久化(撑住容器重启)。
# 这里同时被「主 agent」和「AI 写作 graph」共用 —— LangGraph 用
# (thread_id, checkpoint_ns) 做隔离,session_id 是 uuid,不会撞。
type: sqlite
connection_string: D:\cursor-work\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
connection_string: .deer-flow/data/checkpoints.db
# AI 写作子系统的会话清理配置。
# 业务表(ai_writing_sessions @ MySQL) + checkpointer(checkpoints.db @ SQLite) 联动清理。
# 留意:清理走 cron 表达式调度,默认每天凌晨 3 点跑一次,按 updated_at 一刷拉清。
ai_writing:
cleanup:
enabled: true
retention_days: 7 # 保留 7 天以内的会话,更早的整条删
cron: "0 3 * * *" # 每天 03:00 本地时区
delete_only_finished: false # false=不看状态一刷拉;true=只清 done/error
batch_size: 500 # 单批最多删多少行,避免长事务
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
# 三层覆盖优先级(高->低):
# users/{user_id}/memory_config.json > .deer-flow/memory_user_overrides.json > 此处
memory:
enabled: true
version: v1
provider: hindsight # 'builtin' | 'hindsight'(外部 provider 名)
# builtin 始终启用,这里只决定 external 选哪个
builtin:
@@ -114,7 +151,7 @@ memory:
hindsight: # provider=hindsight 时生效,需另行部署 Hindsight 服务
mode: local_external # 'cloud' | 'local_embedded' | 'local_external'
api_url: http://localhost:8765
api_url: http://47.88.25.99:18888
api_key: "" # 部署 Hindsight 后填,或改成 $HINDSIGHT_API_KEY 并设同名环境变量
# 注意:写成 $VAR 但环境变量未设置会导致配置加载失败
bank_id_template: "deerflow-user-{user_id}"