Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into liuq
This commit is contained in:
commit
9ffe28f049
223 files changed
+28152
-13879
No files matched your search
@@ -58,4 +58,4 @@ ELEMENT_STATE_FILE=/app/backend/.deer-flow/element-bot-token
|
||||
# PYTHON_PACKAGE_INDEX_URL=http://your-python-mirror/simple
|
||||
# PYTHON_EXTRA_INDEX_URL=
|
||||
# PYTHON_TRUSTED_HOST=your-python-mirror
|
||||
MYSQL_DATABASE_URL=mysql+asyncmy://kland:kland%4088124932@192.168.0.123:3306/deerflow?charset=utf8mb4
|
||||
MYSQL_DATABASE_URL=mysql+asyncmy://deerflow:deerflow123@192.168.0.123:3306/deerflow?charset=utf8mb4
|
||||
@@ -187,7 +187,7 @@ Lead-agent middlewares are assembled in strict append order across `packages/har
|
||||
16. **SubagentLimitMiddleware** - Truncates excess `task` tool calls from model response to enforce `MAX_CONCURRENT_SUBAGENTS` limit (optional, if `subagent_enabled`)
|
||||
17. **LoopDetectionMiddleware** - Detects repeated tool-call loops; hard-stop responses clear both structured `tool_calls` and raw provider tool-call metadata before forcing a final text answer
|
||||
18. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last)
|
||||
8a. **ToolMetricsMiddleware** - Appended immediately after `ToolErrorHandlingMiddleware` in `_build_runtime_middlewares`, so it sits _deepest_ in the `wrap_tool_call` chain. Records one `tool_call_metrics` row per tool/skill/MCP invocation — timing duration and bucketing failures into a coarse `error_category` (rate_limited / ip_blocked / auth_error / timeout / network_error / not_found / empty_result / tool_error). It catches both raised exceptions (before #8 converts them) and _self-handled_ errors returned as error `ToolMessage`s (search tools that swallow HTTP 429/403). It also captures the targeted Agent Skill into `tool_call_metrics.skill_name`: the lead agent invokes a skill by `read_file`-ing its `/mnt/skills/<category>/<name>/SKILL.md` path (per the `<skill_system>` prompt block — there is no dedicated "use skill" tool), so `skill_name` is recovered by scanning every tool call's string arguments for the `skills/{public,custom}/{name}/` path shape (`_SKILL_PATH_RE`), plus the explicit `name` argument of `skill_view`/`skill_manage`/`skill_archive`. `skill_view`-specific success detection handles its swallowed failures (`Skill 'x' not found.` returned as a plain string rather than raised). Regression coverage: `tests/test_tool_metrics_middleware.py`. Writes are best-effort fire-and-forget. **Per-run skill merging**: `SqlToolMetricsStore.record` upserts skill rows by `(run_id, skill_name)` instead of inserting one row per call — a skill touched by several tool calls / retries inside one run collapses to a single row. Any success in the run wins (a skill that failed then finally succeeded is recorded as one `success`); a skill that only ever failed yields one `error` row whose `error_message` accumulates every attempt's message; durations are summed. Non-skill tool calls, and skill calls with no `run_id`, keep one row each. The read→merge→write is guarded by a per-event-loop `asyncio.Lock`. Surfaced by the admin `GET /api/admin/tool-metrics` router (which has a `kind=skill` filter and a per-skill `/summary` breakdown).
|
||||
8a. **ToolMetricsMiddleware** - Appended immediately after `ToolErrorHandlingMiddleware` in `_build_runtime_middlewares`, so it sits _deepest_ in the `wrap_tool_call` chain. Records one `tool_call_metrics` row per tool/skill/MCP invocation — timing duration and bucketing failures into a coarse `error_category` (rate*limited / ip_blocked / auth_error / timeout / network_error / not_found / empty_result / tool_error). It catches both raised exceptions (before #8 converts them) and \_self-handled* errors returned as error `ToolMessage`s (search tools that swallow HTTP 429/403). It also captures the targeted Agent Skill into `tool_call_metrics.skill_name`: the lead agent invokes a skill by `read_file`-ing its `/mnt/skills/<category>/<name>/SKILL.md` path (per the `<skill_system>` prompt block — there is no dedicated "use skill" tool), so `skill_name` is recovered by scanning every tool call's string arguments for the `skills/{public,custom}/{name}/` path shape (`_SKILL_PATH_RE`), plus the explicit `name` argument of `skill_view`/`skill_manage`/`skill_archive`. `skill_view`-specific success detection handles its swallowed failures (`Skill 'x' not found.` returned as a plain string rather than raised). Regression coverage: `tests/test_tool_metrics_middleware.py`. Writes are best-effort fire-and-forget. **Per-run skill merging**: `SqlToolMetricsStore.record` upserts skill rows by `(run_id, skill_name)` instead of inserting one row per call — a skill touched by several tool calls / retries inside one run collapses to a single row. Any success in the run wins (a skill that failed then finally succeeded is recorded as one `success`); a skill that only ever failed yields one `error` row whose `error_message` accumulates every attempt's message; durations are summed. Non-skill tool calls, and skill calls with no `run_id`, keep one row each. The read→merge→write is guarded by a per-event-loop `asyncio.Lock`. Surfaced by the admin `GET /api/admin/tool-metrics` router (which has a `kind=skill` filter and a per-skill `/summary` breakdown).
|
||||
19. **PromptPrefixMiddleware** - Injects the admin-configured prompt prefix into the model request only. The prefix is tagged onto the first human message's `additional_kwargs['prompt_prefix']` by the Gateway (`app.gateway.services`) and prepended to content here at call time, so the persisted/displayed message stays prefix-free
|
||||
20. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
|
||||
21. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
|
||||
@@ -249,7 +249,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
|
||||
| **Skills** (`/api/skills`) | `GET /` - list skills (optional `search` fuzzy-name + `tag_id` filters; each item carries `tags` + owner-authored `detail`); `GET /{name}` - details; `PUT /{name}` - update enabled; `PUT /{name}/detail` - set the long-form `detail` text (custom skill: owner or admin; built-in/public: admin only — materializes a `skills` row on demand); `GET /{name}/content` - raw SKILL.md source, restricted to an admin or the publisher (custom-skill owner); other users get 403 and see only description + detail; `POST /install` - install from .skill archive (accepts standard optional frontmatter like `version`, `author`, `compatibility`) |
|
||||
| **Skills** (`/api/skills`) | `GET /` - list skills (optional `search` fuzzy-name + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags` + owner-authored `detail`); `GET /{name}` - details; `PUT /{name}` - update enabled; `PUT /{name}/detail` - set the long-form `detail` text (custom skill: owner or admin; built-in/public: admin only — materializes a `skills` row on demand); `GET /{name}/content` - raw SKILL.md source, restricted to an admin or the publisher (custom-skill owner); other users get 403 and see only description + detail; `POST /install` - install from .skill archive (accepts standard optional frontmatter like `version`, `author`, `compatibility`) |
|
||||
| **Memory** (`/api/memory`) | `GET /` - memory data; `POST /reload` - force reload; `GET /config` - config; `GET /status` - config + data |
|
||||
| **Agents** (`/api/agents`) | `GET /` - list built-in, owned, and published agents (optional `search` + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags`); `POST /` - create user-owned agent; `GET/PUT/DELETE /{id}` - read/update/delete by stable agent id; `GET /check` - validate non-empty display name (duplicates allowed) |
|
||||
| **Agents** (`/api/agents`) | `GET /` - list built-in, owned, and published agents (optional `search` + repeatable `tag_ids` multi-select filter, OR-matched; each item carries `tags` and `featured_order`); `POST /` - create user-owned agent; `GET/PUT/DELETE /{id}` - read/update/delete by stable agent id; `GET /check` - validate non-empty display name (duplicates allowed); `GET /selector` - homepage agent picker (single admin-curated list, visibility-filtered, sorted by `featured_order` asc); `PUT /{id}/featured` - **admin** sets/clears the homepage `featured_order` (must be `published=true` to feature; passing `order: null` clears unconditionally); `PUT /featured/order` - **admin** bulk-reorders the homepage list (body `{agent_ids: [...]}` → indices become 0..N-1) |
|
||||
| **Tags** (`/api/tags`) | `GET /` - list tags (optional `tag_type` filter; open to all); `POST /` - create tag (admin); `PUT /{id}` / `DELETE /{id}` - update/delete tag (admin); `GET /assignments/{target_type}/{target_id}` - tags on a resource; `PUT /assignments/{target_type}/{target_id}` - replace a resource's tags (admin). Tags are typed `agent`/`skill`/`scheduled_task` and may only tag resources of their own type. |
|
||||
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
|
||||
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
|
||||
@@ -276,11 +276,14 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` →
|
||||
|
||||
Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.agents`). `agents.id` is the stable runtime id and the filesystem directory name under `.deer-flow/agents/{id}/`; `agents.name` is display-only, can contain Chinese characters, and is not unique. `user_id IS NULL` marks built-in agents, user-owned agents are visible only to their owner unless `published = true`, and update/delete operations require ownership. The runtime accepts `agent_id` in context/configurable, and non-default `assistant_id` values are treated as agent ids after visibility checks.
|
||||
|
||||
**Homepage selector (agent picker on the new-chat page)** — single admin-curated list backed by `agents.featured_order` (nullable BIGINT). NULL = not featured. Only `published = true` agents can be featured (router enforces on writes); any published agent is eligible regardless of ownership. Cleared with `order: null`. The list rendered to end users is `WHERE featured_order IS NOT NULL` + visibility-filter, sorted by `featured_order ASC`. Admins reorder by sending the desired ids to `PUT /api/agents/featured/order` (assigns indices 0..N-1). BIGINT is used rather than INT so the management UI can seed initial values from `Date.now()` (overflows INT on MySQL).
|
||||
|
||||
### Portable Column Types (`deerflow/persistence/types.py`)
|
||||
|
||||
Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` / `mysql` backends. **All ORM timestamp/JSON columns must use these, not the raw SQLAlchemy types.**
|
||||
|
||||
- **`PortableJSON`** — native `JSON` on SQLite/Postgres; `TEXT` (with manual `json.dumps`/`loads`) on MySQL, where older servers lack JSON support.
|
||||
- **`PortableLongText`** — a large text column: maps to MySQL `LONGTEXT` (4 GB) and plain `TEXT` on SQLite/Postgres (already unbounded). Use for big JSON/markdown blobs that would overflow MySQL's 64 KB `TEXT` cap — e.g. `ai_writing_sessions.transcript`.
|
||||
- **`BeijingDateTime`** — stores timestamps as Beijing (UTC+8) wall-clock time. The app records time with `datetime.now(UTC)`, but MySQL `DATETIME` / SQLite have no timezone and persist whatever wall clock the driver serializes — so a raw `DateTime` column would store UTC and read 8 hours behind Beijing. `BeijingDateTime` converts UTC→Beijing on bind and re-attaches `+08:00` on result, so the database holds Beijing time while Python still gets timezone-aware datetimes (instant comparisons stay correct, including in WHERE clauses). PostgreSQL `timestamptz` stores the instant unambiguously, so values pass through unchanged there. Uses a fixed UTC+8 offset (China has no DST), so no `tzdata` dependency.
|
||||
|
||||
### Sandbox System (`packages/harness/deerflow/sandbox/`)
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
"""AI 写作会话定期清理。
|
||||
|
||||
按 ``config.yaml`` 的 ``ai_writing.cleanup`` 段做联动清理:
|
||||
|
||||
1. 找出 MySQL ``ai_writing_sessions`` 表里 ``updated_at < now - retention_days``
|
||||
的会话 id 列表(分批,避免大事务)
|
||||
2. 对每个 id 调用 LangGraph checkpointer 的 ``adelete_thread()``,删掉
|
||||
SQLite checkpointer 里对应的 thread 状态
|
||||
3. 再把 MySQL 行批量删掉
|
||||
|
||||
整个流程「先删 checkpointer 再删 DB」是有意为之 —— 即便清理被中途中断,
|
||||
也只会出现「checkpointer 删了但 DB 没删」的孤儿,下一轮清理会发现这些
|
||||
DB 行仍然过期,重新尝试清理(checkpointer 删一个不存在的 thread 是 idempotent)。
|
||||
反过来如果先删 DB,万一 checkpointer 删失败,文件里就永久留着不可达的状态。
|
||||
|
||||
多 worker 下用文件锁做 leader election,只有抢到锁的 worker 才会真正跑清理。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── 文件锁 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _acquire_lock(lock_path: Path):
|
||||
"""跨平台非阻塞独占文件锁,多 worker 下做 leader election。
|
||||
|
||||
成功 → 返回持有锁的文件句柄(调用方需保持其存活直到主动释放);
|
||||
抢不到 / 打开失败 → 返回 None。进程退出时 OS 自动释放。
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("ai_writing_cleanup: 打开锁文件失败 path=%s", lock_path)
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
return handle
|
||||
|
||||
|
||||
def _release_lock(handle) -> None:
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
handle.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ── 清理执行 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _delete_checkpoint_thread(checkpointer, thread_id: str) -> None:
|
||||
"""删 checkpointer 里 thread_id 对应的全部状态。失败只记日志,不抛。"""
|
||||
if checkpointer is None:
|
||||
return
|
||||
# LangGraph 异步删法首选 adelete_thread;老版本只有同步 delete_thread。
|
||||
delete_async = getattr(checkpointer, "adelete_thread", None)
|
||||
if delete_async is not None:
|
||||
try:
|
||||
await delete_async(thread_id)
|
||||
return
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: adelete_thread failed thread_id=%s", thread_id, exc_info=True
|
||||
)
|
||||
return
|
||||
delete_sync = getattr(checkpointer, "delete_thread", None)
|
||||
if delete_sync is not None:
|
||||
try:
|
||||
# 同步方法跑在线程池里,避免堵 event loop
|
||||
await asyncio.to_thread(delete_sync, thread_id)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: delete_thread failed thread_id=%s", thread_id, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
async def _get_active_session_ids_langgraph(thread_store) -> set[str]:
|
||||
"""新路径下「正在跑 / 等用户干预」的 thread_id 集合。
|
||||
|
||||
``threads_meta.status`` 由 ``start_run`` / ``run_agent`` 维护:
|
||||
- ``running``:graph 正在跑(节点中)
|
||||
- ``interrupted``:graph 暂停在 interrupt 点等用户回头填卡
|
||||
这两种都视作活跃,**绝不能删**。``idle / success / error`` 才是真正终止状态。
|
||||
|
||||
LangGraph Server 路径 + 老路径的 lead_agent 都用同一份 ``thread_store``,所以
|
||||
这里拿的 active 集合是全部 graph 的;筛 ai_writing 是按 assistant_id 过滤。
|
||||
"""
|
||||
if thread_store is None:
|
||||
return set()
|
||||
try:
|
||||
running = await thread_store.search(status="running", user_id=None, limit=1000)
|
||||
interrupted = await thread_store.search(status="interrupted", user_id=None, limit=1000)
|
||||
except Exception:
|
||||
logger.warning("ai_writing_cleanup: 读取 LangGraph 活动 thread 列表失败", exc_info=True)
|
||||
return set()
|
||||
# 仅关心 assistant_id='ai_writing' 的 thread —— 别的 graph 不归这套清理管
|
||||
active: set[str] = set()
|
||||
for row in running + interrupted:
|
||||
if row.get("assistant_id") == "ai_writing":
|
||||
active.add(row["thread_id"])
|
||||
return active
|
||||
|
||||
|
||||
async def run_cleanup_once(
|
||||
*,
|
||||
retention_days: int,
|
||||
only_finished: bool = False,
|
||||
batch_size: int = 500,
|
||||
checkpointer=None,
|
||||
thread_store=None,
|
||||
) -> dict:
|
||||
"""执行一轮清理。返回统计信息。
|
||||
|
||||
Args:
|
||||
checkpointer: 主 LangGraph checkpointer(``app.state.checkpointer``)。
|
||||
AI 写作的 thread 状态由 LangGraph Server 注入,存在这里 —— admin 路由
|
||||
+ scheduler loop 都会从 ``app.state`` 取到引用传进来;测试场景下传 None
|
||||
会让 checkpoint 删除变成 no-op,但 DB 行仍会被清。
|
||||
thread_store: ``app.state.thread_store``。用来按 ``status in (running,
|
||||
interrupted)`` 查活跃 thread 列表做活跃保护。
|
||||
|
||||
**安全保证**:清理永远跳过仍在跑 / 等用户干预的会话(``threads_meta.status
|
||||
in (running, interrupted)`` + ``assistant_id='ai_writing'``)。即使 admin
|
||||
手动触发 ``retention_days=0`` 也强制生效。
|
||||
|
||||
幂等 + 可重入 —— 多次调用对结果只会越扫越干净。即便清理途中被中断,下次
|
||||
再跑会重新把 DB 里仍然过期的行扫到(checkpointer 删一个已不存在的 thread
|
||||
是 no-op)。
|
||||
"""
|
||||
from deerflow.persistence.ai_writing_sessions import AIWritingSessionRepository
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
logger.info("ai_writing_cleanup: persistence backend=memory,跳过清理")
|
||||
return {"deleted_db": 0, "deleted_checkpoints": 0, "skipped_active": 0}
|
||||
|
||||
repo = AIWritingSessionRepository(sf)
|
||||
active_ids = await _get_active_session_ids_langgraph(thread_store)
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 开始清理 cutoff=%s only_finished=%s batch_size=%d active=%d",
|
||||
cutoff.isoformat(),
|
||||
only_finished,
|
||||
batch_size,
|
||||
len(active_ids),
|
||||
)
|
||||
|
||||
total_db = 0
|
||||
total_cp = 0
|
||||
total_skipped = 0
|
||||
while True:
|
||||
ids = await repo.list_older_than(cutoff, only_finished=only_finished, limit=batch_size)
|
||||
if not ids:
|
||||
break
|
||||
|
||||
# 关键保护:跳过任何仍在内存里跑的会话
|
||||
ids_to_delete = [sid for sid in ids if sid not in active_ids]
|
||||
skipped = len(ids) - len(ids_to_delete)
|
||||
if skipped:
|
||||
logger.warning(
|
||||
"ai_writing_cleanup: 跳过 %d 个仍在跑的会话: %s",
|
||||
skipped,
|
||||
[sid for sid in ids if sid in active_ids],
|
||||
)
|
||||
total_skipped += skipped
|
||||
|
||||
if not ids_to_delete:
|
||||
# 本批全是活动会话 —— 但 list_older_than 是按 updated_at 升序的,
|
||||
# 下一批拿出来的更旧,所以这种情况只可能整个 cutoff 区间都被活动
|
||||
# 会话占满,不用继续往下扫,break
|
||||
break
|
||||
|
||||
# 先删 checkpointer,再删 DB —— 保证「DB 删了 → checkpointer 一定也清干净」
|
||||
for sid in ids_to_delete:
|
||||
await _delete_checkpoint_thread(checkpointer, sid)
|
||||
total_cp += 1
|
||||
|
||||
deleted = await repo.delete_by_ids(ids_to_delete)
|
||||
total_db += deleted
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 本批删 %d 行(DB),%d 个 thread(checkpointer),跳过 %d",
|
||||
deleted,
|
||||
len(ids_to_delete),
|
||||
skipped,
|
||||
)
|
||||
|
||||
# 整批比 batch_size 小 → 没有更多了
|
||||
if len(ids) < batch_size:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 清理完成 total_db=%d total_checkpoints=%d skipped_active=%d",
|
||||
total_db,
|
||||
total_cp,
|
||||
total_skipped,
|
||||
)
|
||||
return {
|
||||
"deleted_db": total_db,
|
||||
"deleted_checkpoints": total_cp,
|
||||
"skipped_active": total_skipped,
|
||||
}
|
||||
|
||||
|
||||
# ── 调度循环 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _next_cron_delay_seconds(cron_expr: str, base: datetime | None = None) -> float:
|
||||
"""根据 cron 表达式算到下一次触发的秒数。
|
||||
|
||||
用 ``datetime.now(UTC)`` 做基准。croniter 接受 naive 时间,这里转成 naive
|
||||
UTC 喂进去;当然如果用户希望按本地时区跑(凌晨 3 点),需要在外层调整 ——
|
||||
我们用本地时区基准,等价于 cron 在本地时区跑。
|
||||
"""
|
||||
base = base or datetime.now()
|
||||
nxt = croniter(cron_expr, base).get_next(datetime)
|
||||
return max(1.0, (nxt - base).total_seconds())
|
||||
|
||||
|
||||
async def cleanup_scheduler_loop(app=None) -> None:
|
||||
"""常驻后台任务:按 cron 调度跑清理。
|
||||
|
||||
多 worker 部署下文件锁做 leader election,没拿到锁的 worker 直接退出。
|
||||
被 cancel 时 CancelledError 不会被 except Exception 吞掉,能干净退出。
|
||||
|
||||
Args:
|
||||
app: FastAPI 实例,用来从 ``app.state`` 拿主 checkpointer / thread_store
|
||||
引用,喂给 ``run_cleanup_once`` —— 新路径下 ai_writing 的 thread 状态
|
||||
在主 checkpointer 里。``app=None`` 时回退到老 checkpointer(仅老路径)。
|
||||
"""
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
try:
|
||||
cfg = get_app_config()
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 加载 app config 失败,调度器退出")
|
||||
return
|
||||
|
||||
raw = getattr(cfg, "ai_writing", None) or {}
|
||||
cleanup_cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
|
||||
if not cleanup_cfg.get("enabled", False):
|
||||
logger.info("ai_writing_cleanup: cleanup.enabled=false,调度器不启动")
|
||||
return
|
||||
|
||||
cron_expr = str(cleanup_cfg.get("cron", "0 3 * * *"))
|
||||
retention_days = int(cleanup_cfg.get("retention_days", 7))
|
||||
only_finished = bool(cleanup_cfg.get("delete_only_finished", False))
|
||||
batch_size = int(cleanup_cfg.get("batch_size", 500))
|
||||
|
||||
# 提前校验 cron 表达式,配错就早 fail 早暴露
|
||||
try:
|
||||
croniter(cron_expr, datetime.now())
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 非法 cron 表达式 %r,调度器退出", cron_expr)
|
||||
return
|
||||
|
||||
lock_handle = _acquire_lock(runtime_home() / ".ai_writing_cleanup.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("ai_writing_cleanup: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"ai_writing_cleanup: 已启动 cron=%r retention_days=%d only_finished=%s batch_size=%d",
|
||||
cron_expr,
|
||||
retention_days,
|
||||
only_finished,
|
||||
batch_size,
|
||||
)
|
||||
try:
|
||||
while True:
|
||||
delay = _next_cron_delay_seconds(cron_expr)
|
||||
logger.info("ai_writing_cleanup: 下一次清理 %.0f 秒后", delay)
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
# 每次循环重新读 app.state —— 不缓存,避免 lifespan 中重启 / 重置
|
||||
# checkpointer 时拿到死引用。app=None 时 main_checkpointer / store
|
||||
# 也为 None,run_cleanup_once 内部走老 fallback。
|
||||
main_checkpointer = getattr(getattr(app, "state", None), "checkpointer", None)
|
||||
main_thread_store = getattr(getattr(app, "state", None), "thread_store", None)
|
||||
await run_cleanup_once(
|
||||
retention_days=retention_days,
|
||||
only_finished=only_finished,
|
||||
batch_size=batch_size,
|
||||
checkpointer=main_checkpointer,
|
||||
thread_store=main_thread_store,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing_cleanup: 本轮清理失败(不影响下一轮)")
|
||||
finally:
|
||||
_release_lock(lock_handle)
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -34,7 +36,9 @@ from app.gateway.routers import (
|
||||
mcp,
|
||||
memory,
|
||||
models,
|
||||
multi_agent,
|
||||
notifications,
|
||||
public_scheduled_tasks,
|
||||
recommended_questions,
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
@@ -244,7 +248,6 @@ async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
|
||||
|
||||
async def _run_db_migrations() -> None:
|
||||
"""Run Alembic migrations at startup so schema changes are applied automatically."""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import command
|
||||
@@ -258,17 +261,50 @@ async def _run_db_migrations() -> None:
|
||||
logger.info("Persistence backend=memory — skipping Alembic migrations")
|
||||
return
|
||||
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
|
||||
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations")
|
||||
return
|
||||
|
||||
db_url = engine.url.render_as_string(hide_password=False).replace("%", "%%")
|
||||
migrations_dir = str(Path(_persistence_pkg.__file__).parent / "migrations")
|
||||
result_queue: queue.Queue[BaseException | None] = queue.Queue(maxsize=1)
|
||||
|
||||
def _run_sync() -> None:
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", migrations_dir)
|
||||
cfg.set_main_option("sqlalchemy.url", db_url)
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", migrations_dir)
|
||||
cfg.set_main_option("sqlalchemy.url", db_url)
|
||||
command.upgrade(cfg, "head")
|
||||
except BaseException as exc:
|
||||
result_queue.put(exc)
|
||||
else:
|
||||
result_queue.put(None)
|
||||
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(None, _run_sync)
|
||||
try:
|
||||
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
|
||||
except ValueError:
|
||||
timeout_s = 60.0
|
||||
|
||||
worker = threading.Thread(target=_run_sync, name="deerflow-alembic-upgrade", daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
deadline = asyncio.get_running_loop().time() + timeout_s
|
||||
while result_queue.empty():
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
raise asyncio.TimeoutError
|
||||
await asyncio.sleep(0.2)
|
||||
result = result_queue.get_nowait()
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(
|
||||
"Database migration did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
|
||||
"to skip startup migrations, or inspect MySQL with SHOW FULL PROCESSLIST.",
|
||||
timeout_s,
|
||||
)
|
||||
return
|
||||
if result is not None:
|
||||
raise result
|
||||
logger.info("Database migrations applied (head)")
|
||||
except Exception:
|
||||
logger.exception("Database migration failed — schema may be out of date")
|
||||
@@ -361,10 +397,67 @@ async def _curator_scheduler_loop() -> None:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
|
||||
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
|
||||
维护 run_id → SSE 订阅者的内存 map。多 worker 下,用户在 worker A 发起
|
||||
``/runs/stream``、稍后在 worker B 调 join/resume,会因 in-process map 互相
|
||||
看不见而 404 / 卡死。
|
||||
|
||||
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
|
||||
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
|
||||
"""
|
||||
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
|
||||
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
|
||||
# / uvicorn 都认)。
|
||||
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
|
||||
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
|
||||
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
|
||||
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
|
||||
web_concurrency,
|
||||
)
|
||||
return
|
||||
|
||||
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
peers = 0
|
||||
for entry in Path("/proc").iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
|
||||
peers += 1
|
||||
if peers > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
|
||||
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
|
||||
"启用 sticky session。",
|
||||
peers,
|
||||
)
|
||||
except Exception:
|
||||
# 检测本身失败不算事 —— 这只是个提醒
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
|
||||
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
|
||||
_warn_if_multi_worker()
|
||||
|
||||
# Load config and check necessary environment variables at startup
|
||||
try:
|
||||
app.state.config = get_app_config()
|
||||
@@ -384,6 +477,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Run Alembic migrations before anything else touches the schema.
|
||||
await _run_db_migrations()
|
||||
|
||||
# 预热 recall_budget cache(从 DB 批量加载,使 get_effective_memory_config 同步可读)
|
||||
try:
|
||||
from deerflow.config.memory_config import load_recall_budget_cache
|
||||
|
||||
_mc_store = getattr(app.state, "memory_config_store", None)
|
||||
if _mc_store is not None:
|
||||
_recall_budgets = await _mc_store.load_all()
|
||||
load_recall_budget_cache(_recall_budgets)
|
||||
logger.info("Memory config cache warmed: %d user override(s)", len(_recall_budgets))
|
||||
except Exception:
|
||||
logger.exception("Memory config cache warm-up failed (non-fatal)")
|
||||
|
||||
# Ensure admin user exists (auto-create on first boot)
|
||||
# Must run AFTER langgraph_runtime so app.state.store is available for thread migration
|
||||
await _ensure_admin_user(app)
|
||||
@@ -402,6 +507,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -439,6 +555,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
# 停止 AI 写作清理调度器
|
||||
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
|
||||
if cleanup_task is not None:
|
||||
cleanup_task.cancel()
|
||||
try:
|
||||
await cleanup_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler shutdown error")
|
||||
|
||||
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
|
||||
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
|
||||
# ``langgraph_runtime()`` 的 contextmanager 统一关闭,无需在这里特殊处理。
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
if scheduler_service is not None:
|
||||
@@ -685,6 +816,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Scheduled task API is mounted at /api/scheduled-tasks
|
||||
app.include_router(scheduled_tasks.router)
|
||||
|
||||
# Public, unauthenticated read-only scheduled task API
|
||||
# (/api/public/scheduled-tasks/*). Lets external systems query tasks and
|
||||
# surface results without a DeerFlow login.
|
||||
app.include_router(public_scheduled_tasks.router)
|
||||
|
||||
# System settings API is mounted at /api/system-settings
|
||||
app.include_router(system_settings.router)
|
||||
|
||||
@@ -725,6 +861,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Stateless Runs API (stream/wait without a pre-existing thread)
|
||||
app.include_router(runs.router)
|
||||
|
||||
# Multi-agent orchestration (coordinator + sub-agent broadcast/dispatch)
|
||||
app.include_router(multi_agent.router)
|
||||
|
||||
@app.get("/health", tags=["health"])
|
||||
async def health_check() -> dict:
|
||||
"""Health check endpoint.
|
||||
|
||||
@@ -37,9 +37,17 @@ def should_check_csrf(request: Request) -> bool:
|
||||
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
|
||||
return False
|
||||
|
||||
path = request.url.path.rstrip("/")
|
||||
path = request.url.path
|
||||
stripped = path.rstrip("/")
|
||||
# Exempt /api/v1/auth/me endpoint
|
||||
if path == "/api/v1/auth/me":
|
||||
if stripped == "/api/v1/auth/me":
|
||||
return False
|
||||
# Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES).
|
||||
# These intentionally have no cookie session, so the Double-Submit Cookie
|
||||
# pattern is both impossible (no token to compare) and unnecessary (no
|
||||
# session for an attacker to ride). Without this exemption every public
|
||||
# PUT/POST returns 403 even though auth was bypassed earlier in the chain.
|
||||
if path.startswith("/api/public/"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@@ -130,14 +130,24 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.thread_share_store = make_thread_share_store(sf)
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions import make_ai_writing_session_store
|
||||
from deerflow.persistence.ai_writing_sessions import (
|
||||
make_ai_writing_session_store,
|
||||
register_default_store as _register_ai_writing_default_store,
|
||||
)
|
||||
|
||||
app.state.ai_writing_session_store = make_ai_writing_session_store(sf)
|
||||
# 让 LangGraph Server 路径下的 AI 写作节点能跨 app/harness 边界拿到 repo。
|
||||
# 老 /start /stream /resume 路由仍走 request.app.state,互不影响。
|
||||
_register_ai_writing_default_store(app.state.ai_writing_session_store)
|
||||
|
||||
from deerflow.persistence.article_types import make_article_type_store
|
||||
|
||||
app.state.article_type_store = make_article_type_store(sf)
|
||||
|
||||
from deerflow.persistence.memory_config import make_user_memory_config_store
|
||||
|
||||
app.state.memory_config_store = make_user_memory_config_store(sf)
|
||||
|
||||
# Run event store (has its own factory with config-driven backend selection)
|
||||
run_events_config = getattr(config, "run_events", None)
|
||||
app.state.run_event_store = make_run_event_store(run_events_config)
|
||||
@@ -261,6 +271,14 @@ def get_article_type_store(request: Request):
|
||||
return getattr(request.app.state, "article_type_store", None)
|
||||
|
||||
|
||||
def get_memory_config_store(request: Request):
|
||||
"""Return the per-user memory config store."""
|
||||
val = getattr(request.app.state, "memory_config_store", None)
|
||||
if val is None:
|
||||
raise HTTPException(status_code=503, detail="Memory config store not available")
|
||||
return val
|
||||
|
||||
|
||||
def get_run_context(request: Request) -> RunContext:
|
||||
"""Build a :class:`RunContext` from ``app.state`` singletons.
|
||||
|
||||
|
||||
@@ -19,14 +19,15 @@ from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import (
|
||||
get_agent_store,
|
||||
get_checkpointer,
|
||||
get_current_user_from_request,
|
||||
get_thread_store,
|
||||
)
|
||||
from deerflow.agents.memory import BuiltinFileProvider
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from app.gateway.routers.memory import _build_v1_provider, _is_v1
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
|
||||
@@ -127,52 +128,289 @@ async def list_user_thread_messages(
|
||||
|
||||
|
||||
class AdminMemoryBucket(BaseModel):
|
||||
name: str = Field(..., description="Bucket name: 'global' or a custom-agent name")
|
||||
name: str = Field(..., description="Bucket name: 'global' or the agent id (used for CRUD)")
|
||||
display_name: str = Field("", description="Human-readable name shown in the UI")
|
||||
data: dict = Field(default_factory=dict, description="Raw memory data for the bucket")
|
||||
|
||||
|
||||
class AdminMemoryEntryAddRequest(BaseModel):
|
||||
content: str = Field(..., min_length=1, description="Entry content to add")
|
||||
agent: str | None = Field(None, description="Agent name; None = global USER.md bucket")
|
||||
|
||||
|
||||
class AdminMemoryEntryReplaceRequest(BaseModel):
|
||||
old_text: str = Field(..., min_length=1, description="Short unique substring to locate the entry")
|
||||
content: str = Field(..., min_length=1, description="New content to replace with")
|
||||
agent: str | None = Field(None, description="Agent name; None = global USER.md bucket")
|
||||
|
||||
|
||||
class AdminMemoryResponse(BaseModel):
|
||||
user_id: str
|
||||
version: str = Field(default="v2", description="记忆版本:v1 或 v2")
|
||||
buckets: list[AdminMemoryBucket] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _read_memory_bucket(user_id: str, agent_id: str | None) -> dict:
|
||||
"""Read one Memory V2 file into a JSON-able ``{"entries": [...]}`` dict.
|
||||
|
||||
``agent_id is None`` -> the user's ``USER.md`` (profile, cross-agent);
|
||||
otherwise that agent's ``MEMORY.md`` (per-agent working notes).
|
||||
"""
|
||||
"""Read one Memory V2 file into a JSON-able ``{"entries": [...]}`` dict."""
|
||||
provider = BuiltinFileProvider()
|
||||
provider.initialize(user_id=user_id, agent_id=agent_id)
|
||||
target = "user" if agent_id is None else "memory"
|
||||
return {"entries": provider.read(target)}
|
||||
|
||||
|
||||
def _read_v1_memory_bucket(user_id: str, agent_id: str | None) -> dict:
|
||||
"""Read one V1 memory.json into its raw dict (facts / user / history)."""
|
||||
provider = _build_v1_provider(user_id, agent_id)
|
||||
return provider.read_v1_data()
|
||||
|
||||
|
||||
@router.get("/{user_id}/memory", response_model=AdminMemoryResponse)
|
||||
async def get_user_memory(user_id: str, request: Request) -> AdminMemoryResponse:
|
||||
"""Return every memory bucket for one user.
|
||||
|
||||
Memory V2 layout: the ``global`` bucket is ``USER.md`` (user profile);
|
||||
each per-agent bucket is that agent's ``MEMORY.md`` working notes.
|
||||
"""
|
||||
"""Return every memory bucket for one user (V1 or V2 depending on config)."""
|
||||
await _require_admin(request)
|
||||
|
||||
buckets: list[AdminMemoryBucket] = [
|
||||
AdminMemoryBucket(name="global", data=_read_memory_bucket(user_id, None))
|
||||
]
|
||||
# Source agents from the database: only agents owned by this specific user.
|
||||
agent_store = get_agent_store(request)
|
||||
all_agents = await agent_store.list_visible(user_id)
|
||||
owned_agents = sorted(
|
||||
[a for a in all_agents if a.get("user_id") == user_id],
|
||||
key=lambda a: (a.get("name") or a["id"]).lower(),
|
||||
)
|
||||
|
||||
agents_root = get_paths().user_dir(user_id) / "agents"
|
||||
if agents_root.is_dir():
|
||||
for entry in sorted(agents_root.iterdir(), key=lambda p: p.name.lower()):
|
||||
if not entry.is_dir() or not (entry / "MEMORY.md").exists():
|
||||
continue
|
||||
if not AGENT_NAME_PATTERN.match(entry.name):
|
||||
continue
|
||||
if _is_v1(user_id):
|
||||
# V1 模式:每个桶读 memory.json,内容为 {facts, user, history}
|
||||
buckets: list[AdminMemoryBucket] = []
|
||||
try:
|
||||
buckets.append(AdminMemoryBucket(name="default", display_name="默认助手", data=_read_v1_memory_bucket(user_id, None)))
|
||||
except Exception:
|
||||
logger.exception("Failed to read V1 global memory for user %s", user_id)
|
||||
buckets.append(AdminMemoryBucket(name="default", display_name="默认助手", data={}))
|
||||
|
||||
for agent in owned_agents:
|
||||
agent_id: str = agent["id"]
|
||||
display_name: str = agent.get("name") or agent_id
|
||||
try:
|
||||
data = _read_memory_bucket(user_id, entry.name)
|
||||
data = _read_v1_memory_bucket(user_id, agent_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to read memory bucket %s for user", entry.name)
|
||||
continue
|
||||
buckets.append(AdminMemoryBucket(name=entry.name, data=data))
|
||||
data = {}
|
||||
buckets.append(AdminMemoryBucket(name=agent_id, display_name=display_name, data=data))
|
||||
|
||||
return AdminMemoryResponse(user_id=user_id, buckets=buckets)
|
||||
return AdminMemoryResponse(user_id=user_id, version="v1", buckets=buckets)
|
||||
|
||||
# V2 模式:读 USER.md / MEMORY.md,内容为 {entries: [...]}
|
||||
buckets = [
|
||||
AdminMemoryBucket(name="default", display_name="默认助手", data=_read_memory_bucket(user_id, None))
|
||||
]
|
||||
for agent in owned_agents:
|
||||
agent_id = agent["id"]
|
||||
display_name = agent.get("name") or agent_id
|
||||
try:
|
||||
data = _read_memory_bucket(user_id, agent_id)
|
||||
except Exception:
|
||||
data = {"entries": []}
|
||||
buckets.append(AdminMemoryBucket(name=agent_id, display_name=display_name, data=data))
|
||||
|
||||
return AdminMemoryResponse(user_id=user_id, version="v2", buckets=buckets)
|
||||
|
||||
|
||||
@router.post("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
|
||||
async def admin_add_memory_entry(
|
||||
user_id: str,
|
||||
body: AdminMemoryEntryAddRequest,
|
||||
request: Request,
|
||||
) -> AdminMemoryBucket:
|
||||
"""Add a memory entry for a user (admin only)."""
|
||||
await _require_admin(request)
|
||||
target = "user" if body.agent is None else "memory"
|
||||
provider = BuiltinFileProvider()
|
||||
provider.initialize(user_id=user_id, agent_id=body.agent)
|
||||
try:
|
||||
result = provider.add(target, body.content)
|
||||
if result is False:
|
||||
raise HTTPException(status_code=400, detail="Failed to add entry (quota exceeded or write error)")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("admin_add_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to write memory entry") from exc
|
||||
bucket_name = "global" if body.agent is None else body.agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, body.agent))
|
||||
|
||||
|
||||
@router.put("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
|
||||
async def admin_replace_memory_entry(
|
||||
user_id: str,
|
||||
body: AdminMemoryEntryReplaceRequest,
|
||||
request: Request,
|
||||
) -> AdminMemoryBucket:
|
||||
"""Replace a memory entry for a user (admin only)."""
|
||||
await _require_admin(request)
|
||||
target = "user" if body.agent is None else "memory"
|
||||
provider = BuiltinFileProvider()
|
||||
provider.initialize(user_id=user_id, agent_id=body.agent)
|
||||
try:
|
||||
result = provider.replace(target, body.old_text, body.content)
|
||||
if result is False:
|
||||
raise HTTPException(status_code=404, detail="Entry not found or write failed")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("admin_replace_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to replace memory entry") from exc
|
||||
bucket_name = "global" if body.agent is None else body.agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, body.agent))
|
||||
|
||||
|
||||
@router.delete("/{user_id}/memory/entries", response_model=AdminMemoryBucket)
|
||||
async def admin_delete_memory_entry(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
old_text: str = Query(..., min_length=1, description="Short unique substring to locate the entry"),
|
||||
agent: str | None = Query(None, description="Agent name; None = global USER.md bucket"),
|
||||
) -> AdminMemoryBucket:
|
||||
"""Delete a memory entry for a user (admin only)."""
|
||||
await _require_admin(request)
|
||||
target = "user" if agent is None else "memory"
|
||||
provider = BuiltinFileProvider()
|
||||
provider.initialize(user_id=user_id, agent_id=agent)
|
||||
try:
|
||||
result = provider.remove(target, old_text)
|
||||
if result is False:
|
||||
raise HTTPException(status_code=404, detail="Entry not found")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("admin_delete_memory_entry failed for user %s: %s", user_id, exc, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to delete memory entry") from exc
|
||||
bucket_name = "global" if agent is None else agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_memory_bucket(user_id, agent))
|
||||
|
||||
|
||||
# ── V1 Memory CRUD (admin) ─────────────────────────────────────────────────
|
||||
|
||||
_V1_USER_FIELDS = {"workContext", "personalContext", "topOfMind"}
|
||||
_V1_HISTORY_FIELDS = {"recentMonths", "earlierContext", "longTermBackground"}
|
||||
|
||||
|
||||
class AdminV1SectionUpdateRequest(BaseModel):
|
||||
section_type: str = Field(..., description="'user' 或 'history'")
|
||||
field: str = Field(..., description="字段名,如 workContext / recentMonths")
|
||||
summary: str = Field(..., description="新的摘要文本")
|
||||
agent: str | None = Field(None, description="Agent id;None 表示用户级全局记忆")
|
||||
|
||||
|
||||
class AdminV1FactAddRequest(BaseModel):
|
||||
content: str = Field(..., min_length=1)
|
||||
category: str = Field(default="context")
|
||||
confidence: float = Field(default=0.8, ge=0.0, le=1.0)
|
||||
agent: str | None = Field(None)
|
||||
|
||||
|
||||
class AdminV1FactUpdateRequest(BaseModel):
|
||||
content: str | None = Field(None)
|
||||
category: str | None = Field(None)
|
||||
confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
agent: str | None = Field(None)
|
||||
|
||||
|
||||
@router.put("/{user_id}/memory/v1/sections", response_model=AdminMemoryBucket)
|
||||
async def admin_update_v1_section(
|
||||
user_id: str,
|
||||
body: AdminV1SectionUpdateRequest,
|
||||
request: Request,
|
||||
) -> AdminMemoryBucket:
|
||||
"""更新 V1 记忆的某个摘要字段(用户画像 / 历史)。"""
|
||||
await _require_admin(request)
|
||||
if body.section_type == "user" and body.field not in _V1_USER_FIELDS:
|
||||
raise HTTPException(status_code=400, detail=f"无效 user 字段:{body.field}")
|
||||
if body.section_type == "history" and body.field not in _V1_HISTORY_FIELDS:
|
||||
raise HTTPException(status_code=400, detail=f"无效 history 字段:{body.field}")
|
||||
if body.section_type not in ("user", "history"):
|
||||
raise HTTPException(status_code=400, detail="section_type 须为 'user' 或 'history'")
|
||||
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage, utc_now_iso_z
|
||||
storage = get_memory_storage()
|
||||
data = storage.load(body.agent, user_id=user_id)
|
||||
data.setdefault(body.section_type, {})[body.field] = {
|
||||
"summary": body.summary,
|
||||
"updatedAt": utc_now_iso_z(),
|
||||
}
|
||||
if not storage.save(data, body.agent, user_id=user_id):
|
||||
raise HTTPException(status_code=500, detail="写入失败")
|
||||
|
||||
bucket_name = "global" if body.agent is None else body.agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
|
||||
|
||||
|
||||
@router.post("/{user_id}/memory/v1/facts", response_model=AdminMemoryBucket)
|
||||
async def admin_add_v1_fact(
|
||||
user_id: str,
|
||||
body: AdminV1FactAddRequest,
|
||||
request: Request,
|
||||
) -> AdminMemoryBucket:
|
||||
"""新增一条 V1 事实记录。"""
|
||||
await _require_admin(request)
|
||||
try:
|
||||
from deerflow.agents.memory.v1.updater import create_memory_fact
|
||||
create_memory_fact(
|
||||
body.content,
|
||||
category=body.category,
|
||||
confidence=body.confidence,
|
||||
agent_name=body.agent,
|
||||
user_id=user_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail="新增失败") from exc
|
||||
bucket_name = "global" if body.agent is None else body.agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
|
||||
|
||||
|
||||
@router.put("/{user_id}/memory/v1/facts/{fact_id}", response_model=AdminMemoryBucket)
|
||||
async def admin_update_v1_fact(
|
||||
user_id: str,
|
||||
fact_id: str,
|
||||
body: AdminV1FactUpdateRequest,
|
||||
request: Request,
|
||||
) -> AdminMemoryBucket:
|
||||
"""更新一条 V1 事实记录(内容 / 类别 / 置信度)。"""
|
||||
await _require_admin(request)
|
||||
try:
|
||||
from deerflow.agents.memory.v1.updater import update_memory_fact
|
||||
update_memory_fact(
|
||||
fact_id,
|
||||
content=body.content,
|
||||
category=body.category,
|
||||
confidence=body.confidence,
|
||||
agent_name=body.agent,
|
||||
user_id=user_id,
|
||||
)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail=f"事实 '{fact_id}' 不存在")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail="更新失败") from exc
|
||||
bucket_name = "global" if body.agent is None else body.agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, body.agent))
|
||||
|
||||
|
||||
@router.delete("/{user_id}/memory/v1/facts/{fact_id}", response_model=AdminMemoryBucket)
|
||||
async def admin_delete_v1_fact(
|
||||
user_id: str,
|
||||
fact_id: str,
|
||||
request: Request,
|
||||
agent: str | None = Query(None, description="Agent id;None 表示用户级全局记忆"),
|
||||
) -> AdminMemoryBucket:
|
||||
"""删除一条 V1 事实记录。"""
|
||||
await _require_admin(request)
|
||||
try:
|
||||
from deerflow.agents.memory.v1.updater import delete_memory_fact
|
||||
delete_memory_fact(fact_id, agent_name=agent, user_id=user_id)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail=f"事实 '{fact_id}' 不存在")
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail="删除失败") from exc
|
||||
bucket_name = "global" if agent is None else agent
|
||||
return AdminMemoryBucket(name=bucket_name, data=_read_v1_memory_bucket(user_id, agent))
|
||||
@@ -45,6 +45,11 @@ class AgentResponse(BaseModel):
|
||||
|
||||
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
|
||||
|
||||
featured_order: int | None = Field(
|
||||
default=None,
|
||||
description="Admin-curated homepage rank. Null = not featured. Lower sorts first.",
|
||||
)
|
||||
|
||||
|
||||
class AgentsListResponse(BaseModel):
|
||||
"""Response model for listing all visible agents."""
|
||||
@@ -55,6 +60,10 @@ class AgentsListResponse(BaseModel):
|
||||
class AgentCreateRequest(BaseModel):
|
||||
"""Request body for creating a custom agent."""
|
||||
|
||||
id: str | None = Field(
|
||||
default=None,
|
||||
description="Optional explicit storage id. Must match the storage id pattern. If omitted, a UUID is generated.",
|
||||
)
|
||||
name: str = Field(..., description="Display name; Chinese and duplicate names are allowed")
|
||||
description: str = Field(default="", description="Agent description")
|
||||
soul: str = Field(default="", description="SOUL.md content - agent personality and behavioral guardrails")
|
||||
@@ -76,6 +85,28 @@ class AgentUpdateRequest(BaseModel):
|
||||
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
|
||||
|
||||
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
"""Response for the homepage agent selector dropdown.
|
||||
|
||||
The list is fully curated by admins (``featured_order`` on each row).
|
||||
Non-admin users get a read-only view of the same list.
|
||||
"""
|
||||
|
||||
agents: list[AgentResponse] = Field(
|
||||
default_factory=list,
|
||||
description="Admin-curated agents, sorted by featured_order ascending.",
|
||||
)
|
||||
|
||||
|
||||
class AgentFeaturedRequest(BaseModel):
|
||||
"""Request body for setting/clearing an agent's homepage featured rank."""
|
||||
|
||||
order: int | None = Field(
|
||||
default=None,
|
||||
description="Featured rank. Lower sorts first. Pass null to remove the agent from the homepage selector.",
|
||||
)
|
||||
|
||||
|
||||
def _require_agents_api_enabled() -> None:
|
||||
"""Reject access unless the custom-agent management API is explicitly enabled."""
|
||||
if not get_agents_api_config().enabled:
|
||||
@@ -179,7 +210,45 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
|
||||
logger.exception("Failed to sync legacy folder-backed agents")
|
||||
|
||||
|
||||
def _record_to_response(record: dict[str, Any], *, include_soul: bool = False) -> AgentResponse:
|
||||
def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
|
||||
"""Lazily restore an agent's on-disk artifacts from its DB record.
|
||||
|
||||
The DB row is authoritative for the agent's existence. If the matching
|
||||
``{base_dir}/agents/{id}/`` directory is missing (data dir moved between
|
||||
machines, filesystem wiped, etc.) we recreate it with a minimal
|
||||
``config.yaml`` + empty ``SOUL.md`` synthesized from the row. This restores
|
||||
a usable agent without requiring the admin to manually delete + recreate.
|
||||
"""
|
||||
agent_id = record.get("id")
|
||||
if not agent_id:
|
||||
return
|
||||
try:
|
||||
agent_dir = get_paths().agent_dir(agent_id)
|
||||
if agent_dir.exists():
|
||||
return
|
||||
logger.warning(
|
||||
"Agent dir missing for '%s' — rebuilding minimal config.yaml + SOUL.md from DB",
|
||||
agent_id,
|
||||
)
|
||||
_write_agent_config(
|
||||
agent_id,
|
||||
name=record.get("name") or agent_id,
|
||||
description=record.get("description") or "",
|
||||
)
|
||||
_write_agent_soul(agent_id, "")
|
||||
except Exception:
|
||||
# Self-heal is best-effort. ``load_agent_config`` already tolerates a
|
||||
# missing dir, so the worst case is the agent runs with default
|
||||
# settings until someone re-edits it.
|
||||
logger.exception("Failed to self-heal agent dir for '%s'", agent_id)
|
||||
|
||||
|
||||
def _record_to_response(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
include_soul: bool = False,
|
||||
) -> AgentResponse:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
soul: str | None = None
|
||||
if include_soul:
|
||||
soul = load_agent_soul(record["id"]) or ""
|
||||
@@ -199,6 +268,7 @@ def _record_to_response(record: dict[str, Any], *, include_soul: bool = False) -
|
||||
model=extras["model"],
|
||||
tool_groups=extras["tool_groups"],
|
||||
skills=extras["skills"],
|
||||
featured_order=record.get("featured_order"),
|
||||
)
|
||||
|
||||
|
||||
@@ -284,6 +354,32 @@ async def check_agent_name(name: str) -> dict[str, Any]:
|
||||
return {"available": True, "name": normalized}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agents/selector",
|
||||
response_model=AgentSelectorResponse,
|
||||
summary="Agent Homepage Selector",
|
||||
description=(
|
||||
"Return the admin-curated agent list shown on the new-chat homepage. "
|
||||
"Sorted by ``featured_order`` ascending. Visibility-filtered, so an "
|
||||
"agent the caller can't see (author un-published) is silently omitted."
|
||||
),
|
||||
)
|
||||
async def get_agent_selector(request: Request) -> AgentSelectorResponse:
|
||||
_require_agents_api_enabled()
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
|
||||
user_id = _current_user_id(request)
|
||||
records = await store.list_featured(user_id)
|
||||
return AgentSelectorResponse(agents=[_record_to_response(r) for r in records])
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to load agent selector: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to load agent selector: {str(e)}") from e
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agents/{agent_id}",
|
||||
response_model=AgentResponse,
|
||||
@@ -297,7 +393,8 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
record = await store.get_visible(agent_id, _current_user_id(request))
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
return _record_to_response(record, include_soul=True)
|
||||
@@ -318,13 +415,19 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
async def create_agent_endpoint(request: AgentCreateRequest, http_request: Request) -> AgentResponse:
|
||||
_require_agents_api_enabled()
|
||||
name = _validate_display_name(request.name)
|
||||
agent_id = uuid4().hex
|
||||
store = get_agent_store(http_request)
|
||||
|
||||
if request.id:
|
||||
agent_id = _validate_storage_id(request.id)
|
||||
if await store.get_any(agent_id) is not None:
|
||||
raise HTTPException(status_code=409, detail=f"Agent id '{agent_id}' already exists")
|
||||
else:
|
||||
agent_id = uuid4().hex
|
||||
agent_dir = get_paths().agent_dir(agent_id)
|
||||
|
||||
try:
|
||||
store = get_agent_store(http_request)
|
||||
if agent_dir.exists():
|
||||
raise HTTPException(status_code=409, detail=f"Generated agent id '{agent_id}' already exists")
|
||||
raise HTTPException(status_code=409, detail=f"Agent id '{agent_id}' already exists")
|
||||
|
||||
_write_agent_config(
|
||||
agent_id,
|
||||
@@ -514,3 +617,91 @@ async def delete_agent(agent_id: str, request: Request) -> None:
|
||||
except Exception as e:
|
||||
logger.error("Failed to delete agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to delete agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.put(
|
||||
"/agents/{agent_id}/featured",
|
||||
response_model=AgentResponse,
|
||||
summary="Set Agent Featured Rank (admin)",
|
||||
description=(
|
||||
"Admin-only. Set or clear the homepage-selector featured rank for any agent. "
|
||||
"Requires the agent to be ``published=true`` when a non-null rank is supplied; "
|
||||
"passing ``order: null`` removes the agent from the selector and is always allowed."
|
||||
),
|
||||
)
|
||||
async def set_agent_featured(
|
||||
agent_id: str,
|
||||
body: AgentFeaturedRequest,
|
||||
request: Request,
|
||||
) -> AgentResponse:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
if not _is_admin_user(request):
|
||||
raise HTTPException(status_code=403, detail="Only system admins can set the homepage featured rank")
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
await _sync_legacy_agents(store)
|
||||
record = await store.get_any(agent_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
|
||||
if body.order is not None and not record.get("published"):
|
||||
raise HTTPException(status_code=409, detail="Only published agents can be featured on the homepage")
|
||||
|
||||
updated = await store.set_featured(agent_id, body.order)
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
logger.info("Set agent '%s' featured_order=%s by admin", agent_id, body.order)
|
||||
return _record_to_response(updated, include_soul=True)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to set featured for agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
|
||||
|
||||
|
||||
class AgentFeaturedReorderRequest(BaseModel):
|
||||
"""Request body for admin to bulk-reorder the homepage featured list."""
|
||||
|
||||
agent_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"Featured agent ids in the desired top→bottom display order. Ids not "
|
||||
"currently featured are silently skipped, so the client can send the "
|
||||
"current visible order without first reconciling with the server."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/agents/featured/order",
|
||||
status_code=204,
|
||||
summary="Reorder Featured Agents (admin)",
|
||||
description=(
|
||||
"Admin-only. Bulk-assign ``featured_order`` to each id in the order received "
|
||||
"(``0`` is topmost). After this, the homepage selector renders agents in "
|
||||
"exactly the supplied order. Ids not already featured are ignored."
|
||||
),
|
||||
)
|
||||
async def reorder_featured_agents(
|
||||
body: AgentFeaturedReorderRequest,
|
||||
request: Request,
|
||||
) -> None:
|
||||
_require_agents_api_enabled()
|
||||
if not _is_admin_user(request):
|
||||
raise HTTPException(status_code=403, detail="Only system admins can reorder the homepage selector")
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
for index, agent_id in enumerate(body.agent_ids):
|
||||
# ``set_featured`` is a no-op for unknown ids — it returns None and
|
||||
# we move on without raising, matching the docstring.
|
||||
await store.set_featured(agent_id, index)
|
||||
logger.info("Admin reordered featured agents (%d ids)", len(body.agent_ids))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to reorder featured agents: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to reorder featured agents: {str(e)}") from e
|
||||
@@ -1,31 +1,26 @@
|
||||
"""AI 写作路由 —— 独立于原有路由,前缀 /api/ai-writing/
|
||||
"""AI 写作路由 —— 业务表 CRUD + transcript 持久化 + admin cleanup。
|
||||
|
||||
写作主流程(``/start /stream /resume /state``)已迁到 LangGraph Server 的
|
||||
``ai_writing`` assistant(前端走 ``useAIWritingStream`` hook + SDK),路由层
|
||||
不再承载 graph 推进,只保留以下「不依赖 graph 状态」的辅助接口:
|
||||
|
||||
POST /api/ai-writing/start 启动写作流程,返回 session_id
|
||||
GET /api/ai-writing/{id}/stream SSE 流(进度/干预/草稿事件)
|
||||
POST /api/ai-writing/{id}/resume 用户干预响应,恢复流程
|
||||
GET /api/ai-writing/{id}/state 查询当前流程状态
|
||||
GET /api/ai-writing/sessions 列出用户的会话历史
|
||||
GET /api/ai-writing/sessions/{id} 获取单个历史会话
|
||||
PATCH /api/ai-writing/sessions/{id} 重命名会话标题
|
||||
DELETE /api/ai-writing/sessions/{id} 删除会话
|
||||
PUT /api/ai-writing/sessions/{id}/transcript 保存完整对话时间线
|
||||
GET /api/ai-writing/article-types 列出所有文章类型
|
||||
POST /api/ai-writing/article-types 新增文章类型
|
||||
PATCH /api/ai-writing/article-types/{id} 更新文章类型
|
||||
DELETE /api/ai-writing/article-types/{id} 删除文章类型
|
||||
POST /api/ai-writing/admin/cleanup (admin)手动清理会话
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -34,35 +29,32 @@ router = APIRouter(prefix="/api/ai-writing", tags=["ai-writing"])
|
||||
|
||||
# ── 请求/响应模型 ─────────────────────────────────────────────────────────────
|
||||
|
||||
class StartWritingRequest(BaseModel):
|
||||
user_intent: str
|
||||
article_type: str = "analysis"
|
||||
target_audience: str = "普通读者"
|
||||
word_count_target: int = 800
|
||||
model_name: str = ""
|
||||
|
||||
|
||||
class StartWritingResponse(BaseModel):
|
||||
session_id: str
|
||||
message: str = "写作流程已启动"
|
||||
|
||||
|
||||
class ResumeRequest(BaseModel):
|
||||
action: str
|
||||
pause_point: str = ""
|
||||
approved_material_ids: list[str] = Field(default_factory=list)
|
||||
extra_keywords: list[str] = Field(default_factory=list)
|
||||
edited_outline: Optional[dict] = None
|
||||
outline_feedback: str = ""
|
||||
user_revision_notes: str = ""
|
||||
override_verdict: bool = False
|
||||
selected_issue_indices: Optional[list[int]] = None
|
||||
|
||||
|
||||
class RenameSessionRequest(BaseModel):
|
||||
title: str
|
||||
|
||||
|
||||
class AdminCleanupRequest(BaseModel):
|
||||
"""手动清理接口的可选覆盖参数。
|
||||
|
||||
全部留空 → 用 config.yaml 的 ai_writing.cleanup 段默认值;显式传值
|
||||
→ 覆盖配置(一般用于运维想验证一个临时阈值,例如清掉「最近 1 小时
|
||||
以前的」会话来模拟整轮调度)。
|
||||
"""
|
||||
retention_days: Optional[int] = Field(default=None, ge=0, description="保留多少天以内的会话;0 = 不保留")
|
||||
only_finished: Optional[bool] = Field(default=None, description="True 时只清 done/error 状态")
|
||||
batch_size: Optional[int] = Field(default=None, ge=1, le=10000, description="单批最多删多少行")
|
||||
|
||||
|
||||
class AdminCleanupResponse(BaseModel):
|
||||
deleted_db: int
|
||||
deleted_checkpoints: int
|
||||
# 因「仍在跑 / 等用户干预」被保护掉的会话数(threads_meta.status in
|
||||
# running/interrupted)。即使 retention_days=0 也不会秒杀用户正在写的稿。
|
||||
skipped_active: int = 0
|
||||
elapsed_seconds: float
|
||||
params: dict
|
||||
|
||||
|
||||
class SaveTranscriptRequest(BaseModel):
|
||||
# 前端组装的完整对话时间线(progressEvents + completedInterventions)
|
||||
transcript: dict
|
||||
@@ -113,140 +105,7 @@ class SessionResponse(BaseModel):
|
||||
updated_at: str
|
||||
|
||||
|
||||
# ── 内部状态 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# session_id → asyncio.Queue
|
||||
_event_queues: dict[str, asyncio.Queue] = {}
|
||||
|
||||
# session_id → model_name(resume 时复用)
|
||||
_session_model_names: dict[str, str] = {}
|
||||
|
||||
# session_id → AIWritingSessionRepository(for DB saves)
|
||||
_session_repos: dict[str, object] = {}
|
||||
|
||||
# session_id → user_id (str | None, already resolved)
|
||||
_session_user_ids: dict[str, str | None] = {}
|
||||
|
||||
# session_id → accumulated intervention records
|
||||
_session_interventions: dict[str, list] = {}
|
||||
|
||||
# session_id → True if session ended with error
|
||||
_session_error_flags: dict[str, bool] = {}
|
||||
|
||||
# session_id → valid_article_types(list[dict] with key/label)
|
||||
_session_article_types: dict[str, list[dict]] = {}
|
||||
|
||||
# session_id → pending await_user event tuple for SSE reconnect re-delivery
|
||||
# asyncio.Queue is consume-once; if the old SSE generator dequeued the await_user event
|
||||
# before the client disconnected, the event is lost from the queue. We save a copy here
|
||||
# so that a new SSE connection (e.g., after user clicks "继续") can re-inject it.
|
||||
_session_pending_pause: dict[str, tuple] = {}
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _sse(event: str, data: dict) -> str:
|
||||
return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
|
||||
|
||||
def _ai_writing_config() -> dict:
|
||||
"""读取 config.yaml 的 ai_writing 段(含素材检索 provider 配置)。"""
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
aw = getattr(get_app_config(), "ai_writing", None)
|
||||
if isinstance(aw, dict):
|
||||
return aw
|
||||
return dict(aw) if aw else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _make_config(session_id: str, model_name: str = "", valid_article_types: list[dict] | None = None) -> dict:
|
||||
return {
|
||||
"configurable": {
|
||||
"thread_id": session_id,
|
||||
"model_name": model_name or _session_model_names.get(session_id, ""),
|
||||
"ai_writing_config": _ai_writing_config(),
|
||||
# 把 SSE 队列注入,供草稿/大纲节点直接推流式 chunk
|
||||
"sse_queue": _event_queues.get(session_id),
|
||||
"sse_session_id": session_id,
|
||||
# 动态文章类型列表,intent_parser 从此处读取有效枚举
|
||||
"valid_article_types": valid_article_types or _session_article_types.get(session_id),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _truncate_title(text: str, max_len: int = 50) -> str:
|
||||
text = text.strip()
|
||||
return text[:max_len] + ("…" if len(text) > max_len else "")
|
||||
|
||||
|
||||
# ── 路由 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/start", response_model=StartWritingResponse)
|
||||
async def start_writing(body: StartWritingRequest, request: Request):
|
||||
"""启动写作流程,返回 session_id。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_article_type_store, get_current_user
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
_event_queues[session_id] = queue
|
||||
_session_model_names[session_id] = body.model_name
|
||||
_session_interventions[session_id] = []
|
||||
|
||||
# 加载动态文章类型列表,注入给 intent_parser 节点
|
||||
article_type_store = get_article_type_store(request)
|
||||
valid_article_types: list[dict] | None = None
|
||||
if article_type_store is not None:
|
||||
try:
|
||||
records = await article_type_store.list()
|
||||
if records:
|
||||
valid_article_types = [{"key": r["key"], "label": r["label"]} for r in records]
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to load article types for session=%s", session_id)
|
||||
_session_article_types[session_id] = valid_article_types or []
|
||||
|
||||
# 持久化会话记录(DB 可能未配置,则跳过)
|
||||
repo = get_ai_writing_session_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
if repo is not None:
|
||||
try:
|
||||
title = _truncate_title(body.user_intent)
|
||||
await repo.create(
|
||||
id=session_id,
|
||||
title=title,
|
||||
user_intent=body.user_intent,
|
||||
user_id=user_id,
|
||||
)
|
||||
_session_repos[session_id] = repo
|
||||
_session_user_ids[session_id] = user_id
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to create session record: session=%s", session_id)
|
||||
|
||||
initial_state = {
|
||||
"user_intent": body.user_intent,
|
||||
"article_type": body.article_type,
|
||||
"target_audience": body.target_audience,
|
||||
"word_count_target": body.word_count_target,
|
||||
"revision_count": 0,
|
||||
"max_revisions": 3,
|
||||
"status": "researching",
|
||||
"drafts": [],
|
||||
"progress_events": [],
|
||||
"material_package": None,
|
||||
"current_outline": None,
|
||||
"current_draft": None,
|
||||
"review_result": None,
|
||||
"last_user_intervention": None,
|
||||
}
|
||||
config = _make_config(session_id, body.model_name, valid_article_types=valid_article_types)
|
||||
|
||||
asyncio.create_task(_stream_graph_segment(session_id, initial_state, config, AI_WRITING_GRAPH))
|
||||
return StartWritingResponse(session_id=session_id)
|
||||
|
||||
|
||||
@router.get("/sessions", response_model=list[SessionResponse])
|
||||
async def list_sessions(request: Request):
|
||||
"""列出当前用户的 AI 写作历史会话。"""
|
||||
@@ -306,19 +165,27 @@ async def delete_session(session_id: str, request: Request):
|
||||
|
||||
@router.put("/sessions/{session_id}/transcript")
|
||||
async def save_transcript(session_id: str, body: SaveTranscriptRequest, request: Request):
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。"""
|
||||
"""保存会话的完整对话时间线,供历史会话回显出与实时对话一致的富时间线。
|
||||
|
||||
时间线持久化属于「锦上添花」功能:即便保存失败(如内网部署缺少 transcript
|
||||
列、或 MySQL TEXT 列超长),也绝不能阻断写作流程。任何异常都吞掉并返回
|
||||
success=False,由前端忽略即可。
|
||||
"""
|
||||
from app.gateway.deps import get_ai_writing_session_store, get_current_user
|
||||
|
||||
repo = get_ai_writing_session_store(request)
|
||||
if repo is None:
|
||||
return {"success": False}
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
return {"success": False, "reason": "store_unavailable"}
|
||||
try:
|
||||
user_id = await get_current_user(request)
|
||||
updated = await repo.update(session_id, user_id=user_id, transcript=body.transcript)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save transcript: session=%s", session_id)
|
||||
return {"success": False, "reason": "save_failed"}
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
# 会话不存在或不属于当前用户:同样不抛 500,返回 success=False 即可
|
||||
return {"success": False, "reason": "session_not_found"}
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@router.get("/article-types", response_model=list[ArticleTypeResponse])
|
||||
async def list_article_types(request: Request):
|
||||
"""列出所有文章类型,按 sort_order 升序排列。"""
|
||||
@@ -392,249 +259,109 @@ async def delete_article_type(article_type_id: str, request: Request):
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@router.get("/{session_id}/stream")
|
||||
async def stream_progress(session_id: str):
|
||||
"""SSE 流:推送进度事件、干预请求、草稿内容。"""
|
||||
return StreamingResponse(
|
||||
_sse_generator(session_id),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
|
||||
# ── 管理接口 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/{session_id}/resume")
|
||||
async def resume_writing(session_id: str, body: ResumeRequest, request: Request):
|
||||
"""用户提交干预决策,恢复暂停的流程。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
from app.gateway.deps import get_current_user
|
||||
async def _require_admin(request: Request) -> None:
|
||||
"""放行 admin / auth-disabled,其余 403。
|
||||
|
||||
if session_id not in _event_queues:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
|
||||
config = _make_config(session_id)
|
||||
intervention = {
|
||||
"action": body.action,
|
||||
"approvedMaterialIds": body.approved_material_ids,
|
||||
"extraKeywords": body.extra_keywords,
|
||||
"editedOutline": body.edited_outline,
|
||||
"outlineFeedback": body.outline_feedback,
|
||||
"userRevisionNotes": body.user_revision_notes,
|
||||
"overrideVerdict": body.override_verdict,
|
||||
"selectedIssueIndices": body.selected_issue_indices,
|
||||
}
|
||||
|
||||
# Record this intervention for session history
|
||||
record = {
|
||||
"pausePoint": body.pause_point,
|
||||
"action": body.action,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
if session_id in _session_interventions:
|
||||
_session_interventions[session_id].append(record)
|
||||
repo = _session_repos.get(session_id)
|
||||
user_id = _session_user_ids.get(session_id)
|
||||
if repo is not None:
|
||||
try:
|
||||
await repo.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
completed_interventions=_session_interventions[session_id],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save intervention: session=%s", session_id)
|
||||
|
||||
# Clear pending pause so SSE reconnect won't re-inject the stale await_user event
|
||||
_session_pending_pause.pop(session_id, None)
|
||||
|
||||
asyncio.create_task(
|
||||
_stream_graph_segment(session_id, Command(resume=intervention), config, AI_WRITING_GRAPH)
|
||||
)
|
||||
return {"message": "已提交,流程恢复中"}
|
||||
|
||||
|
||||
@router.get("/{session_id}/state")
|
||||
async def get_state(session_id: str):
|
||||
"""查询当前流程状态快照。"""
|
||||
from deerflow.agents.ai_writing.graph import AI_WRITING_GRAPH
|
||||
|
||||
config = _make_config(session_id)
|
||||
snapshot = await AI_WRITING_GRAPH.aget_state(config)
|
||||
if not snapshot or not snapshot.values:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
return snapshot.values
|
||||
|
||||
|
||||
# ── 内部辅助 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _stream_graph_segment(
|
||||
session_id: str,
|
||||
input_or_command,
|
||||
config: dict,
|
||||
graph,
|
||||
):
|
||||
"""运行一个 Graph 片段(初始运行或 resume),将输出推入队列。
|
||||
遇到 interrupt 时停止(不关闭队列),等待下次 resume。
|
||||
Graph 结束时关闭队列(放入 None 哨兵)。
|
||||
与 scheduled_tasks._require_admin 同套语义:auth_disabled 模式下
|
||||
get_optional_user_from_request 返回 None,此时放行(开发环境一致体验)。
|
||||
"""
|
||||
queue = _event_queues.get(session_id)
|
||||
if queue is None:
|
||||
return
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
|
||||
reached_end = False
|
||||
is_error = False
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="AI 写作清理管理接口仅限管理员")
|
||||
|
||||
|
||||
def _resolve_cleanup_params(override: AdminCleanupRequest) -> dict:
|
||||
"""合并 config.yaml 默认值 + 请求显式覆盖。
|
||||
|
||||
显式传 0 / False 也算覆盖(用 ``is not None`` 判定),方便测「保留 0 天 =
|
||||
全删」这种极端场景。
|
||||
"""
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
raw = getattr(get_app_config(), "ai_writing", None) or {}
|
||||
cfg = (raw.get("cleanup") if isinstance(raw, dict) else None) or {}
|
||||
|
||||
return {
|
||||
"retention_days": (
|
||||
override.retention_days if override.retention_days is not None
|
||||
else int(cfg.get("retention_days", 7))
|
||||
),
|
||||
"only_finished": (
|
||||
override.only_finished if override.only_finished is not None
|
||||
else bool(cfg.get("delete_only_finished", False))
|
||||
),
|
||||
"batch_size": (
|
||||
override.batch_size if override.batch_size is not None
|
||||
else int(cfg.get("batch_size", 500))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/admin/cleanup", response_model=AdminCleanupResponse)
|
||||
async def admin_cleanup(
|
||||
body: AdminCleanupRequest,
|
||||
request: Request,
|
||||
) -> AdminCleanupResponse:
|
||||
"""**仅管理员** 手动触发一次会话清理(不等凌晨 3 点定时调度)。
|
||||
|
||||
用途:
|
||||
- 验证 cleanup 配置改动是否生效
|
||||
- 内网紧急清理(磁盘满 / MySQL 行数过多)
|
||||
- 调试:传 ``retention_days: 0`` 一刷拉清空,方便重新跑回归
|
||||
|
||||
所有参数留空 → 沿用 config.yaml 的 ``ai_writing.cleanup`` 段默认。显式
|
||||
传值会覆盖配置但**不修改 config.yaml**(一次性参数,下次定时调度还是用
|
||||
配置文件的值)。
|
||||
"""
|
||||
import time
|
||||
|
||||
from app.gateway.ai_writing_cleanup import run_cleanup_once
|
||||
|
||||
await _require_admin(request)
|
||||
params = _resolve_cleanup_params(body)
|
||||
|
||||
# 透传主 LangGraph checkpointer / thread_store —— 新路径下 ai_writing 的
|
||||
# thread 状态在主 checkpointer 里,thread_store 用来按 status 过滤活跃 thread。
|
||||
# getattr 嵌套防御:测试用 stub Request 时 ``request.app`` 可能不存在。
|
||||
_app_state = getattr(getattr(request, "app", None), "state", None)
|
||||
main_checkpointer = getattr(_app_state, "checkpointer", None)
|
||||
main_thread_store = getattr(_app_state, "thread_store", None)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
async for chunk in graph.astream(input_or_command, config=config, stream_mode="updates"):
|
||||
node_name = next(iter(chunk))
|
||||
node_output = chunk[node_name]
|
||||
|
||||
# interrupt() 触发
|
||||
if node_name == "__interrupt__":
|
||||
# node_output 是 (Interrupt(...),) 元组
|
||||
interrupt_list = node_output if isinstance(node_output, (list, tuple)) else [node_output]
|
||||
for intr in interrupt_list:
|
||||
interrupt_data = intr.value if hasattr(intr, "value") else intr
|
||||
event_tuple = ("await_user", interrupt_data)
|
||||
await queue.put(event_tuple)
|
||||
# Save for SSE reconnect re-delivery: if the old generator consumed this
|
||||
# event before the client disconnected, a new connection would block forever.
|
||||
_session_pending_pause[session_id] = event_tuple
|
||||
return # 等待 resume,不关闭队列
|
||||
|
||||
if not isinstance(node_output, dict):
|
||||
continue
|
||||
|
||||
# 推送 progress_events
|
||||
for evt in node_output.get("progress_events") or []:
|
||||
await queue.put((_event_type(evt), {**evt, "session_id": session_id}))
|
||||
|
||||
# 素材就绪
|
||||
if pkg := node_output.get("material_package"):
|
||||
await queue.put(("materials_ready", {
|
||||
"session_id": session_id,
|
||||
"keywords": pkg.get("keywords", []),
|
||||
"materials": pkg.get("materials", []),
|
||||
"summary": pkg.get("summary", ""),
|
||||
}))
|
||||
|
||||
# 大纲就绪
|
||||
if outline := node_output.get("current_outline"):
|
||||
await queue.put(("outline_ready", {"session_id": session_id, "outline": outline}))
|
||||
|
||||
# 草稿就绪 — 保存到 DB
|
||||
if draft := node_output.get("current_draft"):
|
||||
draft_title = draft.get("title", "")
|
||||
draft_markdown = draft.get("full_markdown", "")
|
||||
await queue.put(("draft_ready", {
|
||||
"session_id": session_id,
|
||||
"title": draft_title,
|
||||
"markdown": draft_markdown,
|
||||
"revision_count": draft.get("revision_count", 0),
|
||||
}))
|
||||
repo = _session_repos.get(session_id)
|
||||
user_id = _session_user_ids.get(session_id)
|
||||
if repo is not None:
|
||||
try:
|
||||
await repo.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
status="draft_ready",
|
||||
title=draft_title or _truncate_title(
|
||||
next(iter(_session_interventions.get(session_id, [{}])), {}).get("action", "草稿"),
|
||||
),
|
||||
draft_title=draft_title,
|
||||
draft_markdown=draft_markdown,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save draft: session=%s", session_id)
|
||||
|
||||
# 编辑审核就绪 — 保存到 DB
|
||||
if review := node_output.get("review_result"):
|
||||
await queue.put(("review_ready", {"session_id": session_id, "review_result": review}))
|
||||
repo = _session_repos.get(session_id)
|
||||
user_id = _session_user_ids.get(session_id)
|
||||
if repo is not None:
|
||||
try:
|
||||
await repo.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
status="review_ready",
|
||||
review_result=review,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to save review: session=%s", session_id)
|
||||
|
||||
# 状态 done:Graph 结束
|
||||
if node_output.get("status") == "done":
|
||||
reached_end = True
|
||||
|
||||
# astream 正常结束 = Graph 执行到 END
|
||||
reached_end = True
|
||||
await queue.put(("done", {"session_id": session_id, "message": "写作完成"}))
|
||||
|
||||
result = await run_cleanup_once(
|
||||
**params,
|
||||
checkpointer=main_checkpointer,
|
||||
thread_store=main_thread_store,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing graph segment failed: session=%s", session_id)
|
||||
await queue.put(("error", {"session_id": session_id, "message": str(e)}))
|
||||
is_error = True
|
||||
reached_end = True
|
||||
finally:
|
||||
if reached_end:
|
||||
# 更新最终状态到 DB
|
||||
repo = _session_repos.get(session_id)
|
||||
user_id = _session_user_ids.get(session_id)
|
||||
if repo is not None:
|
||||
try:
|
||||
await repo.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
status="error" if is_error else "done",
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("ai_writing: failed to update final status: session=%s", session_id)
|
||||
logger.exception("ai_writing admin cleanup failed: params=%s", params)
|
||||
# 抛 500 是合理的 —— 管理员触发清理失败需要看到失败原因,跟「业务
|
||||
# 接口被失败拖累」是两回事
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"清理执行失败: {type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
await queue.put(None) # 哨兵,通知 SSE 生成器关闭
|
||||
_event_queues.pop(session_id, None)
|
||||
_session_model_names.pop(session_id, None)
|
||||
_session_repos.pop(session_id, None)
|
||||
_session_user_ids.pop(session_id, None)
|
||||
_session_interventions.pop(session_id, None)
|
||||
_session_error_flags.pop(session_id, None)
|
||||
_session_article_types.pop(session_id, None)
|
||||
_session_pending_pause.pop(session_id, None)
|
||||
|
||||
|
||||
def _event_type(evt: dict) -> str:
|
||||
t = evt.get("type", "progress")
|
||||
# 过滤掉已在上方单独推送的事件类型,避免重复
|
||||
if t in ("materials_ready", "outline_ready", "draft_ready", "review_ready", "await_user",
|
||||
"draft_chunk", "outline_chunk"):
|
||||
return "step_done"
|
||||
return t
|
||||
|
||||
|
||||
async def _sse_generator(session_id: str) -> AsyncGenerator[str, None]:
|
||||
"""从队列消费事件,推送为 SSE 帧。"""
|
||||
# 等待队列创建(最多 5 秒)
|
||||
for _ in range(50):
|
||||
if session_id in _event_queues:
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
yield _sse("error", {"message": "Session not found"})
|
||||
return
|
||||
|
||||
queue = _event_queues[session_id]
|
||||
yield _sse("connected", {"session_id": session_id})
|
||||
|
||||
# Re-inject the saved await_user event if the queue is empty and a pending pause exists.
|
||||
# This covers the case where the old SSE generator dequeued the await_user event and
|
||||
# yielded it to a disconnecting client — the event is then lost from the consume-once queue.
|
||||
if queue.empty() and session_id in _session_pending_pause:
|
||||
await queue.put(_session_pending_pause[session_id])
|
||||
|
||||
while True:
|
||||
item = await queue.get()
|
||||
if item is None:
|
||||
break
|
||||
event_type, data = item
|
||||
yield _sse(event_type, data)
|
||||
await asyncio.sleep(0)
|
||||
elapsed = time.monotonic() - started
|
||||
logger.info(
|
||||
"ai_writing admin cleanup done: params=%s deleted_db=%d deleted_cp=%d elapsed=%.2fs",
|
||||
params,
|
||||
result["deleted_db"],
|
||||
result["deleted_checkpoints"],
|
||||
elapsed,
|
||||
)
|
||||
return AdminCleanupResponse(
|
||||
deleted_db=result["deleted_db"],
|
||||
deleted_checkpoints=result["deleted_checkpoints"],
|
||||
skipped_active=result.get("skipped_active", 0),
|
||||
elapsed_seconds=round(elapsed, 3),
|
||||
params=params,
|
||||
)
|
||||
@@ -21,15 +21,16 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_memory_config_store
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.memory_config import (
|
||||
USER_WRITABLE_FIELDS,
|
||||
delete_user_memory_override,
|
||||
delete_user_memory_override_db,
|
||||
get_effective_memory_config,
|
||||
save_user_memory_override,
|
||||
save_user_memory_override_db,
|
||||
)
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
@@ -38,7 +39,7 @@ router = APIRouter(prefix="/api", tags=["memory"])
|
||||
|
||||
|
||||
# 表示"全局桶 (无 agent)"的 agent 取值。
|
||||
_GLOBAL_AGENT_VALUES: frozenset[str] = frozenset({"", "global", "__global__"})
|
||||
_GLOBAL_AGENT_VALUES: frozenset[str] = frozenset({"", "global", "__global__", "default"})
|
||||
|
||||
|
||||
def _resolve_agent_name(agent: str | None) -> str | None:
|
||||
@@ -72,20 +73,38 @@ _AgentQuery = Query(
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class MemoryV2DataResponse(BaseModel):
|
||||
"""Memory V2 builtin provider 的数据视图。"""
|
||||
class MemoryFact(BaseModel):
|
||||
"""V1 记忆中的单条事实。"""
|
||||
|
||||
user_entries: list[str] = Field(default_factory=list, description="USER.md 条目 (用户画像)")
|
||||
memory_entries: list[str] = Field(default_factory=list, description="MEMORY.md 条目 (agent 工作笔记)")
|
||||
user_usage: str = Field(default="", description="USER.md 字符用量,如 '12% — 165/1375 字符'")
|
||||
memory_usage: str = Field(default="", description="MEMORY.md 字符用量")
|
||||
id: str = Field(default="", description="事实 UUID")
|
||||
content: str = Field(default="", description="事实文本")
|
||||
category: str = Field(default="knowledge", description="事实类别")
|
||||
confidence: float = Field(default=0.0, description="置信度 0-1")
|
||||
createdAt: str = Field(default="", description="创建时间 ISO 格式")
|
||||
sourceError: str = Field(default="", description="纠错类事实:被纠正的原错误描述")
|
||||
|
||||
|
||||
class MemoryV2DataResponse(BaseModel):
|
||||
"""Memory 数据视图 —— V1/V2 共用,多余字段各版本置空。"""
|
||||
|
||||
# V2 字段
|
||||
user_entries: list[str] = Field(default_factory=list, description="V2: USER.md 条目 (用户画像)")
|
||||
memory_entries: list[str] = Field(default_factory=list, description="V2: MEMORY.md 条目 (agent 工作笔记)")
|
||||
user_usage: str = Field(default="", description="字符/计数用量描述")
|
||||
memory_usage: str = Field(default="", description="字符/计数用量描述")
|
||||
|
||||
# V1 字段
|
||||
facts: list[MemoryFact] = Field(default_factory=list, description="V1: facts 列表 (含置信度)")
|
||||
user_context: dict[str, str] = Field(default_factory=dict, description="V1: userContext 摘要")
|
||||
history: dict[str, str] = Field(default_factory=dict, description="V1: history 历史摘要")
|
||||
|
||||
|
||||
class MemoryV2ConfigResponse(BaseModel):
|
||||
"""Memory V2 生效配置 (含三层覆盖合并结果) + 用户可改字段白名单。"""
|
||||
"""Memory 生效配置 (含三层覆盖合并结果) + 用户可改字段白名单 + 版本号。"""
|
||||
|
||||
config: dict[str, Any] = Field(..., description="按 per-user 覆盖合并后的生效配置")
|
||||
writable_fields: list[str] = Field(..., description="用户可在运行时覆盖的字段 (点号路径)")
|
||||
version: str = Field(default="v2", description="当前记忆版本:v1 或 v2")
|
||||
|
||||
|
||||
class MemoryConfigPatchRequest(BaseModel):
|
||||
@@ -133,6 +152,33 @@ class MemorySearchResponse(BaseModel):
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _is_v1(user_id: str) -> bool:
|
||||
"""判断当前用户是否处于 V1 记忆模式。"""
|
||||
config = get_effective_memory_config(user_id)
|
||||
return getattr(config, "version", "v2") == "v1"
|
||||
|
||||
|
||||
def _build_v1_provider(user_id: str, agent_name: str | None = None):
|
||||
"""构建并初始化 V1BuiltinProvider。"""
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
config = get_effective_memory_config(user_id)
|
||||
v1_cfg = config.v1
|
||||
provider = V1BuiltinProvider(
|
||||
max_facts=v1_cfg.max_facts,
|
||||
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
|
||||
debounce_seconds=v1_cfg.debounce_seconds,
|
||||
model_name=v1_cfg.model_name,
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
provider.initialize(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name,
|
||||
base_dir=str(get_paths().base_dir),
|
||||
)
|
||||
return provider
|
||||
|
||||
|
||||
def _build_builtin_provider(user_id: str, agent_name: str | None):
|
||||
"""按生效配置构建并初始化一个 V2 builtin provider。"""
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
@@ -160,7 +206,7 @@ def _validate_v2_target(target: str) -> str:
|
||||
|
||||
|
||||
def _v2_data_response(provider) -> MemoryV2DataResponse:
|
||||
"""从 builtin provider 读出当前数据,组装成 MemoryV2DataResponse。"""
|
||||
"""从 V2 builtin provider 读出当前数据。"""
|
||||
return MemoryV2DataResponse(
|
||||
user_entries=provider.read("user"),
|
||||
memory_entries=provider.read("memory"),
|
||||
@@ -169,6 +215,49 @@ def _v2_data_response(provider) -> MemoryV2DataResponse:
|
||||
)
|
||||
|
||||
|
||||
def _v1_data_response(provider) -> MemoryV2DataResponse:
|
||||
"""从 V1 provider 读出当前数据,转换为统一响应格式。
|
||||
|
||||
支持新格式 {user: {workContext: {summary, updatedAt}}} 和老格式 {userContext: {workContext: str}}。
|
||||
"""
|
||||
data = provider.read_v1_data()
|
||||
raw_facts = data.get("facts", [])
|
||||
facts = [
|
||||
MemoryFact(
|
||||
id=str(f.get("id", "")),
|
||||
content=str(f.get("content", "")),
|
||||
category=str(f.get("category", "knowledge")),
|
||||
confidence=float(f.get("confidence", 0)),
|
||||
createdAt=str(f.get("createdAt", "")),
|
||||
sourceError=str(f.get("sourceError", "")),
|
||||
)
|
||||
for f in raw_facts
|
||||
]
|
||||
|
||||
def _extract_section(section: dict) -> dict[str, str]:
|
||||
"""从 {key: {summary: str, updatedAt: str}} 或 {key: str} 中提取摘要字符串。"""
|
||||
result = {}
|
||||
for k, v in section.items():
|
||||
if isinstance(v, dict):
|
||||
summary = v.get("summary", "")
|
||||
else:
|
||||
summary = str(v)
|
||||
if summary:
|
||||
result[k] = summary
|
||||
return result
|
||||
|
||||
# 新格式用 "user" key,老格式用 "userContext" key(已由 storage 迁移,此处兜底)
|
||||
ctx_raw = data.get("user") or data.get("userContext") or {}
|
||||
history_raw = data.get("history") or {}
|
||||
|
||||
return MemoryV2DataResponse(
|
||||
facts=sorted(facts, key=lambda f: f.confidence, reverse=True),
|
||||
user_context=_extract_section(ctx_raw),
|
||||
history=_extract_section(history_raw),
|
||||
memory_usage=provider.usage_text(),
|
||||
)
|
||||
|
||||
|
||||
def _raise_if_write_failed(result: dict) -> None:
|
||||
"""builtin provider 写操作失败时抛 400。"""
|
||||
if not result.get("success"):
|
||||
@@ -183,33 +272,43 @@ def _raise_if_write_failed(result: dict) -> None:
|
||||
@router.get(
|
||||
"/memory/v2",
|
||||
response_model=MemoryV2DataResponse,
|
||||
summary="Get Memory V2 Data",
|
||||
description="读取 Memory V2 builtin provider 的 USER.md / MEMORY.md 条目。",
|
||||
summary="Get Memory Data",
|
||||
description="读取记忆数据。V1 模式返回 facts 列表;V2 模式返回 USER.md / MEMORY.md 条目。",
|
||||
)
|
||||
async def get_memory_v2(agent: str | None = _AgentQuery) -> MemoryV2DataResponse:
|
||||
"""读取当前用户的 Memory V2 本地记忆。"""
|
||||
"""读取当前用户的本地记忆。"""
|
||||
agent_name = _resolve_agent_name(agent)
|
||||
user_id = get_effective_user_id()
|
||||
try:
|
||||
if _is_v1(user_id):
|
||||
provider = _build_v1_provider(user_id, agent_name)
|
||||
return _v1_data_response(provider)
|
||||
provider = _build_builtin_provider(user_id, agent_name)
|
||||
return _v2_data_response(provider)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=500, detail="Failed to read memory V2 data.") from exc
|
||||
raise HTTPException(status_code=500, detail="Failed to read memory data.") from exc
|
||||
|
||||
|
||||
def _assert_not_v1(user_id: str) -> None:
|
||||
"""V1 模式下写入操作返回 400。"""
|
||||
if _is_v1(user_id):
|
||||
raise HTTPException(status_code=400, detail="V1 模式下记忆由系统自动提取,不支持手动修改。")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/memory/v2/entries",
|
||||
response_model=MemoryV2DataResponse,
|
||||
summary="Add Memory V2 Entry",
|
||||
description="手动往 USER.md / MEMORY.md 新增一条记忆条目。",
|
||||
description="手动往 USER.md / MEMORY.md 新增一条记忆条目 (仅 V2 模式)。",
|
||||
)
|
||||
async def add_memory_v2_entry(
|
||||
request: MemoryEntryCreateRequest,
|
||||
agent: str | None = _AgentQuery,
|
||||
) -> MemoryV2DataResponse:
|
||||
"""手动新增一条 V2 记忆条目。"""
|
||||
_validate_v2_target(request.target)
|
||||
user_id = get_effective_user_id()
|
||||
_assert_not_v1(user_id)
|
||||
_validate_v2_target(request.target)
|
||||
try:
|
||||
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
|
||||
result = provider.add(request.target, request.content)
|
||||
@@ -225,15 +324,16 @@ async def add_memory_v2_entry(
|
||||
"/memory/v2/entries",
|
||||
response_model=MemoryV2DataResponse,
|
||||
summary="Replace Memory V2 Entry",
|
||||
description="用 old_text 定位一条记忆条目并替换为新内容。",
|
||||
description="用 old_text 定位一条记忆条目并替换为新内容 (仅 V2 模式)。",
|
||||
)
|
||||
async def replace_memory_v2_entry(
|
||||
request: MemoryEntryReplaceRequest,
|
||||
agent: str | None = _AgentQuery,
|
||||
) -> MemoryV2DataResponse:
|
||||
"""手动替换一条 V2 记忆条目。"""
|
||||
_validate_v2_target(request.target)
|
||||
user_id = get_effective_user_id()
|
||||
_assert_not_v1(user_id)
|
||||
_validate_v2_target(request.target)
|
||||
try:
|
||||
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
|
||||
result = provider.replace(request.target, request.old_text, request.content)
|
||||
@@ -249,7 +349,7 @@ async def replace_memory_v2_entry(
|
||||
"/memory/v2/entries",
|
||||
response_model=MemoryV2DataResponse,
|
||||
summary="Delete Memory V2 Entry",
|
||||
description="用 old_text 定位并删除一条记忆条目。",
|
||||
description="用 old_text 定位并删除一条记忆条目 (仅 V2 模式)。",
|
||||
)
|
||||
async def delete_memory_v2_entry(
|
||||
target: str = Query(..., description="memory 或 user"),
|
||||
@@ -257,8 +357,9 @@ async def delete_memory_v2_entry(
|
||||
agent: str | None = _AgentQuery,
|
||||
) -> MemoryV2DataResponse:
|
||||
"""手动删除一条 V2 记忆条目。"""
|
||||
_validate_v2_target(target)
|
||||
user_id = get_effective_user_id()
|
||||
_assert_not_v1(user_id)
|
||||
_validate_v2_target(target)
|
||||
try:
|
||||
provider = _build_builtin_provider(user_id, _resolve_agent_name(agent))
|
||||
result = provider.remove(target, old_text)
|
||||
@@ -288,6 +389,7 @@ async def get_memory_v2_config() -> MemoryV2ConfigResponse:
|
||||
return MemoryV2ConfigResponse(
|
||||
config=config.model_dump(),
|
||||
writable_fields=sorted(USER_WRITABLE_FIELDS),
|
||||
version=getattr(config, "version", "v2"),
|
||||
)
|
||||
|
||||
|
||||
@@ -295,20 +397,24 @@ async def get_memory_v2_config() -> MemoryV2ConfigResponse:
|
||||
"/memory/v2/config",
|
||||
response_model=MemoryV2ConfigResponse,
|
||||
summary="Patch Memory V2 Config",
|
||||
description="把白名单字段写入当前用户的 per-user 配置覆盖。非白名单字段被忽略。",
|
||||
description="把 hindsight.recall_budget 写入当前用户的 per-user 配置覆盖 (DB 存储)。",
|
||||
)
|
||||
async def patch_memory_v2_config(request: MemoryConfigPatchRequest) -> MemoryV2ConfigResponse:
|
||||
"""更新当前用户的 per-user 记忆配置覆盖。"""
|
||||
async def patch_memory_v2_config(request: MemoryConfigPatchRequest, req: Request) -> MemoryV2ConfigResponse:
|
||||
"""更新当前用户的 per-user 记忆配置覆盖 (仅 hindsight.recall_budget)。"""
|
||||
user_id = get_effective_user_id()
|
||||
store = get_memory_config_store(req)
|
||||
try:
|
||||
save_user_memory_override(user_id, request.overrides)
|
||||
except OSError as exc:
|
||||
await save_user_memory_override_db(user_id, request.overrides, store)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=500, detail="Failed to save memory config override.") from exc
|
||||
|
||||
config = get_effective_memory_config(user_id)
|
||||
return MemoryV2ConfigResponse(
|
||||
config=config.model_dump(),
|
||||
writable_fields=sorted(USER_WRITABLE_FIELDS),
|
||||
version=getattr(config, "version", "v2"),
|
||||
)
|
||||
|
||||
|
||||
@@ -317,24 +423,27 @@ async def patch_memory_v2_config(request: MemoryConfigPatchRequest) -> MemoryV2C
|
||||
response_model=MemoryV2ConfigResponse,
|
||||
summary="Delete Memory V2 Config Override",
|
||||
description=(
|
||||
"撤销当前用户的 per-user 配置覆盖。传 ?field=injection.enabled 只撤销该字段;"
|
||||
"不传 field 则清空整个 per-user 覆盖。"
|
||||
"撤销当前用户的 per-user 配置覆盖。传 ?field=hindsight.recall_budget 只撤销该字段;"
|
||||
"不传 field 则清空全部覆盖。"
|
||||
),
|
||||
)
|
||||
async def delete_memory_v2_config(
|
||||
req: Request,
|
||||
field: str | None = Query(None, description="要撤销的点号路径字段;省略则清空全部覆盖"),
|
||||
) -> MemoryV2ConfigResponse:
|
||||
"""撤销当前用户的 per-user 记忆配置覆盖。"""
|
||||
user_id = get_effective_user_id()
|
||||
store = get_memory_config_store(req)
|
||||
try:
|
||||
delete_user_memory_override(user_id, field)
|
||||
except OSError as exc:
|
||||
await delete_user_memory_override_db(user_id, field, store)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=500, detail="Failed to delete memory config override.") from exc
|
||||
|
||||
config = get_effective_memory_config(user_id)
|
||||
return MemoryV2ConfigResponse(
|
||||
config=config.model_dump(),
|
||||
writable_fields=sorted(USER_WRITABLE_FIELDS),
|
||||
version=getattr(config, "version", "v2"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,818 @@
|
||||
"""Multi-agent orchestration endpoints.
|
||||
|
||||
Wraps the legacy ``newpython/OUT.py`` demo into the Gateway. Provides:
|
||||
|
||||
* ``POST /api/multi-agent/init`` — create a coordinator agent and per-sub-agent
|
||||
threads. Returns the ``{agent_name: thread_id}`` map the client passes back
|
||||
on subsequent calls.
|
||||
* ``POST /api/multi-agent/run/stream`` — drive a *leader* or *special* agent
|
||||
via SSE. The leader dispatches work to sub-agents through the
|
||||
``agent_orchestration`` skill; sub-agents execute and broadcast their result.
|
||||
|
||||
Internal calls loop back to the same Gateway over HTTP, reusing the caller's
|
||||
``Authorization`` header — so the demo's hard-coded ``get_token("ji")`` is gone.
|
||||
The model used by both the coordinator agent and every streamed run is taken
|
||||
from the request's ``model`` field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/multi-agent", tags=["multi-agent"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loopback helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _loopback_base(request: Request) -> str:
|
||||
return f"{request.url.scheme}://{request.url.netloc}"
|
||||
|
||||
|
||||
def _auth_headers(request: Request) -> dict[str, str]:
|
||||
token = request.headers.get("authorization")
|
||||
if not token:
|
||||
raise HTTPException(status_code=401, detail="Missing Authorization header")
|
||||
return {"Authorization": token, "Content-Type": "application/json"}
|
||||
|
||||
|
||||
async def _create_thread(client: httpx.AsyncClient, base: str, headers: dict[str, str]) -> str:
|
||||
resp = await client.post(f"{base}/api/threads", json={}, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["thread_id"]
|
||||
|
||||
|
||||
async def _get_agent(client: httpx.AsyncClient, base: str, headers: dict[str, str], agent_id: str) -> dict[str, Any]:
|
||||
resp = await client.get(f"{base}/api/agents/{agent_id}", headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def _create_agent(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
model: str | None,
|
||||
skills: list[str] | None,
|
||||
soul: str,
|
||||
agent_id: str | None = None,
|
||||
) -> tuple[bool, dict[str, Any] | str]:
|
||||
payload: dict[str, Any] = {"name": name, "description": description, "soul": soul}
|
||||
if agent_id is not None:
|
||||
payload["id"] = agent_id
|
||||
if model is not None:
|
||||
payload["model"] = model
|
||||
if skills is not None:
|
||||
payload["skills"] = skills
|
||||
resp = await client.post(f"{base}/api/agents", json=payload, headers=headers)
|
||||
if resp.status_code == 201:
|
||||
return True, resp.json()
|
||||
return False, resp.text
|
||||
|
||||
|
||||
async def _append_thread_message(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
thread_id: str,
|
||||
content: str,
|
||||
) -> None:
|
||||
state_url = f"{base}/api/threads/{thread_id}/state"
|
||||
get_resp = await client.get(state_url, headers=headers)
|
||||
if get_resp.status_code != 200:
|
||||
logger.warning("get state %s failed: %s %s", thread_id, get_resp.status_code, get_resp.text[:200])
|
||||
return
|
||||
state = get_resp.json()
|
||||
values = state.setdefault("values", {})
|
||||
messages = values.setdefault("messages", [])
|
||||
if not isinstance(messages, list):
|
||||
messages = []
|
||||
values["messages"] = messages
|
||||
messages.append({"type": "human", "content": content})
|
||||
post_resp = await client.post(state_url, json=state, headers=headers)
|
||||
if post_resp.status_code >= 400:
|
||||
logger.warning("update state %s failed: %s %s", thread_id, post_resp.status_code, post_resp.text[:200])
|
||||
|
||||
|
||||
async def _broadcast(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
message: str,
|
||||
agent_threads: dict[str, str],
|
||||
skip: list[str],
|
||||
) -> None:
|
||||
targets = [tid for name, tid in agent_threads.items() if name not in skip]
|
||||
if not targets:
|
||||
return
|
||||
await asyncio.gather(
|
||||
*(_append_thread_message(client, base, headers, tid, message) for tid in targets),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
|
||||
_TOOL_CALL_PATTERN = re.compile(r'\.py\s+"([^"]*)"\s+"([^"]*)"')
|
||||
|
||||
# Mirrors the upstream agent-id validator in the LangGraph runtime. Reject early
|
||||
# so callers get a clear 400 here instead of a stream that aborts with HTTP 422
|
||||
# from the loopback /api/threads/.../runs/stream endpoint.
|
||||
_AGENT_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
|
||||
|
||||
def _validate_agent_id(value: str, field: str) -> str:
|
||||
if not _AGENT_ID_PATTERN.match(value):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"{field} '{value}' is not a valid agent id. "
|
||||
f"Must match ^[A-Za-z0-9_-]+$ (use the hyphen-case id, not the display name)."
|
||||
),
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _extract_dispatch(tool_call: dict[str, Any]) -> tuple[str, str]:
|
||||
"""Return ``(agent_name, task)`` parsed from a leader tool call.
|
||||
|
||||
Two formats are accepted:
|
||||
|
||||
1. **Structured** — ``{"name": "agent_orchestration", "args": {"agent_name": ..., "task": ...}}``.
|
||||
This is what most LLMs emit when told to "use the agent_orchestration skill",
|
||||
and the form the roundtable coordinator's SOUL prompt elicits.
|
||||
|
||||
2. **Legacy bash argv** — ``{"name": "bash", "args": {"command": '... .py "X" "Y" ...'}}``.
|
||||
Kept as a fallback so older skills that dispatch through bash still work.
|
||||
"""
|
||||
name = (tool_call.get("name") or "").lower()
|
||||
args = tool_call.get("args")
|
||||
|
||||
# Format 1: structured agent_orchestration call.
|
||||
if name == "agent_orchestration" and isinstance(args, dict):
|
||||
agent_name = str(args.get("agent_name") or "").strip()
|
||||
task = str(args.get("task") or "").strip()
|
||||
if agent_name and task:
|
||||
return _strip_display_suffix(agent_name), task
|
||||
|
||||
# Format 2: legacy bash + .py "X" "Y".
|
||||
match = _TOOL_CALL_PATTERN.search(str(args))
|
||||
if match:
|
||||
return _strip_display_suffix(match.group(1)), match.group(2)
|
||||
|
||||
return "", ""
|
||||
|
||||
|
||||
def _strip_display_suffix(agent_name: str) -> str:
|
||||
"""Tolerate models that paste the human-readable suffix into agent_name.
|
||||
|
||||
The coordinator SOUL lists each seat as ``agent_name: roundtable-x(中文):...``
|
||||
so weaker models sometimes echo the entire ``roundtable-x(中文)`` label
|
||||
back as the dispatch target. Strip everything from the first non-id
|
||||
character onward so we recover the bare hyphen-case id; if what's left
|
||||
still violates the id pattern we hand it back untouched and let the
|
||||
upstream validator reject it.
|
||||
"""
|
||||
head = re.split(r"[^A-Za-z0-9_-]", agent_name, maxsplit=1)[0]
|
||||
return head or agent_name
|
||||
|
||||
|
||||
def _flatten_content(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
parts.append(str(item.get("text", "")))
|
||||
else:
|
||||
parts.append(str(item))
|
||||
return "".join(parts)
|
||||
return str(content)
|
||||
|
||||
|
||||
def _run_payload(
|
||||
*,
|
||||
agent_name: str,
|
||||
model_name: str,
|
||||
thread_id: str,
|
||||
new_message: str,
|
||||
excluded_tools: list[str],
|
||||
skill_stop_names: list[str],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"input": {
|
||||
"messages": [
|
||||
{
|
||||
"type": "human",
|
||||
"content": [{"type": "text", "text": new_message}],
|
||||
"additional_kwargs": {},
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {"recursion_limit": 1000},
|
||||
"context": {
|
||||
"agent_name": agent_name,
|
||||
"model_name": model_name,
|
||||
"mode": "pro",
|
||||
"reasoning_effort": "medium",
|
||||
"thinking_enabled": True,
|
||||
"is_plan_mode": False,
|
||||
"subagent_enabled": False,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
"stream_mode": ["messages-tuple", "values"],
|
||||
"stream_subgraphs": True,
|
||||
"stream_resumable": True,
|
||||
"assistant_id": "lead_agent",
|
||||
"on_disconnect": "continue",
|
||||
"excluded_tools": excluded_tools,
|
||||
"skill_stop_names": skill_stop_names,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Init endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/init")
|
||||
async def init_multi_agent(request: Request, payload: dict = Body(...)) -> StreamingResponse:
|
||||
"""Streaming /init.
|
||||
|
||||
Emits text/event-stream frames as each seat/coordinator becomes ready, so
|
||||
the frontend can visualize the parallel bring-up rather than blocking on
|
||||
one bulk JSON. Frames:
|
||||
|
||||
data: {"event":"seat_ready","agent_name":"...","thread_id":"...","display_name":"...","description":"..."}
|
||||
data: {"event":"coordinator_ready","main_agent_name":"...","thread_id":"..."}
|
||||
data: {"event":"init_done","main_agent_name":"...","thread_ids":{...}}
|
||||
data: {"event":"error","detail":"..."} # on failure
|
||||
|
||||
Validation errors (missing agent_names, bad id pattern) still raise
|
||||
HTTPException synchronously so the connection fails fast with a real 4xx
|
||||
instead of a 200 SSE that just carries an error frame.
|
||||
"""
|
||||
agent_names_in = payload.get("agent_names") or []
|
||||
agent_names: list[str] = [str(name).lower() for name in agent_names_in]
|
||||
main_agent_name: str = str(payload.get("main_agent_name") or "main-agent").lower()
|
||||
model: str | None = payload.get("model")
|
||||
|
||||
if not agent_names:
|
||||
raise HTTPException(status_code=400, detail="agent_names is required")
|
||||
|
||||
for name in agent_names:
|
||||
_validate_agent_id(name, "agent_names entry")
|
||||
_validate_agent_id(main_agent_name, "main_agent_name")
|
||||
|
||||
base = _loopback_base(request)
|
||||
headers = _auth_headers(request)
|
||||
|
||||
def _frame(payload: dict[str, Any]) -> str:
|
||||
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
||||
|
||||
async def stream_generator() -> AsyncIterator[str]:
|
||||
async with httpx.AsyncClient(timeout=None) as client:
|
||||
try:
|
||||
# Per-seat prepare task — same body as before, but we no longer
|
||||
# await them with asyncio.gather. Instead, asyncio.as_completed
|
||||
# surfaces them in REAL completion order so we can emit a
|
||||
# seat_ready frame the moment each finishes.
|
||||
async def _prepare_sub_agent(sub_id: str) -> tuple[str, str, str, str]:
|
||||
info = await _get_agent(client, base, headers, sub_id)
|
||||
display_name = str(info.get("name") or sub_id)
|
||||
description = info.get("description", "") or ""
|
||||
thread_id = await _create_thread(client, base, headers)
|
||||
return sub_id.lower(), thread_id, display_name, description
|
||||
|
||||
tasks = [asyncio.create_task(_prepare_sub_agent(sid)) for sid in agent_names]
|
||||
prepared: list[tuple[str, str, str, str]] = []
|
||||
try:
|
||||
for fut in asyncio.as_completed(tasks):
|
||||
sid, thread_id, display_name, description = await fut
|
||||
prepared.append((sid, thread_id, display_name, description))
|
||||
yield _frame({
|
||||
"event": "seat_ready",
|
||||
"agent_name": sid,
|
||||
"thread_id": thread_id,
|
||||
"display_name": display_name,
|
||||
"description": description,
|
||||
})
|
||||
except Exception:
|
||||
# Cancel any still-pending sibling so we don't leak tasks
|
||||
# past the AsyncClient close.
|
||||
for t in tasks:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
raise
|
||||
|
||||
agent_threads: dict[str, str] = {sid: thread_id for sid, thread_id, _, _ in prepared}
|
||||
descriptions: list[str] = [
|
||||
f"- agent_name: {sid}({display_name}):{description}"
|
||||
for sid, _, display_name, description in prepared
|
||||
]
|
||||
|
||||
soul = (
|
||||
"#核心定位:\n"
|
||||
"用途:根据用户的任务目标,调用其他智能体干活,只负责协调汇总,"
|
||||
"当你觉得已有信息支持回答用户问题时候,停止调度,并输出最后的结果\n"
|
||||
"#调度方法:\n"
|
||||
"通过使用agent_orchestration技能调用其他智能体干活,使用技能时传入agent_name和需要做的活。"
|
||||
"注意:agent_name 必须严格使用下方列出的英文 id(例如 roundtable-intelligence),"
|
||||
"不要把括号里的中文显示名拼进 agent_name 里。\n"
|
||||
"#可调度智能体:\n"
|
||||
+ "\n".join(descriptions)
|
||||
)
|
||||
|
||||
coord_thread = await _create_thread(client, base, headers)
|
||||
agent_threads[main_agent_name] = coord_thread
|
||||
ok, result = await _create_agent(
|
||||
client,
|
||||
base,
|
||||
headers,
|
||||
name=main_agent_name,
|
||||
description="你是总控控制协调智能体,负责协调其他智能体干活",
|
||||
model=model,
|
||||
skills=["agent_orchestration"],
|
||||
soul=soul,
|
||||
agent_id=main_agent_name,
|
||||
)
|
||||
if not ok and isinstance(result, str) and "exists" not in result:
|
||||
yield _frame({"event": "error", "detail": f"create coordinator agent failed: {result}"})
|
||||
return
|
||||
|
||||
yield _frame({
|
||||
"event": "coordinator_ready",
|
||||
"main_agent_name": main_agent_name,
|
||||
"thread_id": coord_thread,
|
||||
})
|
||||
yield _frame({
|
||||
"event": "init_done",
|
||||
"main_agent_name": main_agent_name,
|
||||
"thread_ids": agent_threads,
|
||||
})
|
||||
except HTTPException as exc:
|
||||
logger.warning("multi-agent init HTTPException: %s %s", exc.status_code, exc.detail)
|
||||
yield _frame({"event": "error", "detail": f"[{exc.status_code}] {exc.detail}"})
|
||||
except Exception as exc:
|
||||
logger.exception("multi-agent init unexpected failure")
|
||||
yield _frame({"event": "error", "detail": f"unexpected: {exc!r}"})
|
||||
|
||||
return StreamingResponse(
|
||||
stream_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stream endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _stream_upstream(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
thread_id: str,
|
||||
body: dict[str, Any],
|
||||
) -> AsyncIterator[tuple[str | None, list[str] | None]]:
|
||||
"""Yield (raw_line, None) for each SSE line, then ``(None, captured_data_lines)`` once."""
|
||||
|
||||
url = f"{base}/api/threads/{thread_id}/runs/stream"
|
||||
captured: list[str] = []
|
||||
t_send = time.perf_counter()
|
||||
async with client.stream("POST", url, json=body, headers=headers) as resp:
|
||||
t_headers = time.perf_counter()
|
||||
if resp.status_code != 200:
|
||||
body_text = (await resp.aread()).decode("utf-8", "replace")[:1000]
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"upstream {url} returned HTTP {resp.status_code}: {body_text}",
|
||||
)
|
||||
agent_for_log = (body.get("context") or {}).get("agent_name") or thread_id
|
||||
logger.info(
|
||||
"[upstream-timing] %s headers_in=%.3fs",
|
||||
agent_for_log,
|
||||
t_headers - t_send,
|
||||
)
|
||||
first_line = True
|
||||
async for line in resp.aiter_lines():
|
||||
if not line:
|
||||
continue
|
||||
if first_line:
|
||||
first_line = False
|
||||
logger.info(
|
||||
"[upstream-timing] %s first_line=%.3fs (LangGraph runtime startup + LLM first byte)",
|
||||
agent_for_log,
|
||||
time.perf_counter() - t_headers,
|
||||
)
|
||||
yield line, None
|
||||
if line.startswith("data:"):
|
||||
captured.append(line)
|
||||
yield None, captured
|
||||
|
||||
|
||||
def _parse_last_messages(captured: list[str]) -> list[dict[str, Any]]:
|
||||
"""Return the full ``messages`` list from the most recent ``values`` snapshot.
|
||||
|
||||
The upstream sends the final ``values`` frame second-to-last (the very last
|
||||
line is the ``end`` event with ``null``). We tolerate trailing non-JSON
|
||||
frames (e.g. ``data: null``) by scanning backwards for the first frame that
|
||||
successfully parses as a state object with a ``messages`` key.
|
||||
"""
|
||||
if not captured:
|
||||
raise HTTPException(status_code=502, detail="upstream produced no data lines")
|
||||
|
||||
for raw_line in reversed(captured):
|
||||
raw = raw_line[5:].strip() if raw_line.startswith("data:") else raw_line.strip()
|
||||
if not raw or raw == "null":
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(parsed, dict) and "messages" in parsed:
|
||||
messages = parsed.get("messages") or []
|
||||
if isinstance(messages, list):
|
||||
return messages
|
||||
|
||||
raise HTTPException(status_code=502, detail="no parseable values frame in upstream stream")
|
||||
|
||||
|
||||
def _find_trailing_ai_text(messages: list[dict[str, Any]], *, exclude_text: str = "") -> str:
|
||||
"""Return the text content of the latest AIMessage with no tool_calls.
|
||||
|
||||
Used by the clarification branch: ClarificationMiddleware ends the run
|
||||
after emitting the structured ToolMessage, but the model often produces a
|
||||
richer human-readable follow-up AIMessage right before the interrupt.
|
||||
Prefer that text over the brief preamble that accompanied the
|
||||
ask_clarification tool_call. When the only trailing AIMessage matches the
|
||||
preamble (``exclude_text``), return empty so the caller can fall back to
|
||||
the question itself.
|
||||
"""
|
||||
for msg in reversed(messages):
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
if (msg.get("type") or "").lower() not in ("ai", "aimessage", "aimessagechunk"):
|
||||
continue
|
||||
if msg.get("tool_calls"):
|
||||
continue
|
||||
text = _flatten_content(msg.get("content", "")).strip()
|
||||
if not text:
|
||||
continue
|
||||
if exclude_text and text == exclude_text.strip():
|
||||
continue
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _find_dispatch_message(messages: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Locate the latest AIMessage that carries dispatch tool calls.
|
||||
|
||||
The very last message can be a skill-stop ToolMessage injected by
|
||||
``SkillStopMiddleware`` when it intercepts the dispatch, or a clarification
|
||||
ToolMessage. Either way, the actionable tool_calls live on the AIMessage
|
||||
*before* it, so we walk backwards until we hit one with ``tool_calls``.
|
||||
Returns the latest AIMessage when none of them carry tool_calls so the
|
||||
caller can still extract its text content.
|
||||
"""
|
||||
last_ai: dict[str, Any] = {}
|
||||
for msg in reversed(messages):
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
if (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
|
||||
if not last_ai:
|
||||
last_ai = msg
|
||||
if msg.get("tool_calls"):
|
||||
return msg
|
||||
return last_ai
|
||||
|
||||
|
||||
async def _leader_run(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
agent_threads: dict[str, str],
|
||||
agent_name: str,
|
||||
new_message: str,
|
||||
skill_stop_names: list[str],
|
||||
model_name: str,
|
||||
) -> AsyncIterator[str | dict[str, Any]]:
|
||||
# Fine-grained timing for the leader path. The user reported the
|
||||
# coordinator "stalls before output" — we log each major await so we can
|
||||
# see whether the wait is in pre-broadcast scheduling, upstream stream
|
||||
# setup, first-token latency, or post-stream parsing.
|
||||
t0 = time.perf_counter()
|
||||
|
||||
pre_broadcast_task = asyncio.create_task(
|
||||
_broadcast(client, base, headers, "总控智能体收到用户消息:" + new_message, agent_threads, [agent_name])
|
||||
)
|
||||
thread_id = agent_threads[agent_name]
|
||||
effective_stop_names = list({"agent_orchestration", *skill_stop_names})
|
||||
body = _run_payload(
|
||||
agent_name=agent_name,
|
||||
model_name=model_name,
|
||||
thread_id=thread_id,
|
||||
new_message=new_message,
|
||||
excluded_tools=["web_search"],
|
||||
skill_stop_names=effective_stop_names,
|
||||
)
|
||||
t_payload_built = time.perf_counter()
|
||||
logger.info(
|
||||
"[leader-timing] %s payload_built=%.3fs (broadcast running in background)",
|
||||
agent_name,
|
||||
t_payload_built - t0,
|
||||
)
|
||||
|
||||
captured: list[str] = []
|
||||
first_line_ts: float | None = None
|
||||
line_count = 0
|
||||
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
|
||||
if line is not None:
|
||||
if first_line_ts is None:
|
||||
first_line_ts = time.perf_counter()
|
||||
logger.info(
|
||||
"[leader-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
|
||||
agent_name,
|
||||
first_line_ts - t_payload_built,
|
||||
first_line_ts - t0,
|
||||
)
|
||||
line_count += 1
|
||||
yield line
|
||||
elif done is not None:
|
||||
captured = done
|
||||
t_stream_done = time.perf_counter()
|
||||
logger.info(
|
||||
"[leader-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
|
||||
agent_name,
|
||||
t_stream_done - (first_line_ts or t_payload_built),
|
||||
line_count,
|
||||
t_stream_done - t0,
|
||||
)
|
||||
|
||||
messages = _parse_last_messages(captured)
|
||||
# Pre-broadcast must finish before we yield the final status frame on any
|
||||
# branch — the seats that get dispatched next rely on having seen the
|
||||
# original user message. The upstream LLM stream usually takes several
|
||||
# seconds, so the broadcast has typically already completed and this
|
||||
# await is effectively free.
|
||||
t_before_broadcast_await = time.perf_counter()
|
||||
try:
|
||||
await pre_broadcast_task
|
||||
except Exception as exc:
|
||||
logger.warning("pre-broadcast failed: %s", exc)
|
||||
t_after_broadcast_await = time.perf_counter()
|
||||
logger.info(
|
||||
"[leader-timing] %s pre_broadcast_await=%.3fs (>0 means broadcast was still in flight after LLM finished)",
|
||||
agent_name,
|
||||
t_after_broadcast_await - t_before_broadcast_await,
|
||||
)
|
||||
|
||||
dispatch_msg = _find_dispatch_message(messages)
|
||||
leader_text = _flatten_content(dispatch_msg.get("content", ""))
|
||||
tool_calls = dispatch_msg.get("tool_calls") or []
|
||||
|
||||
# Clarification branch: the coordinator called ask_clarification. The
|
||||
# ClarificationMiddleware intercepts with Command(goto=END) and stashes
|
||||
# the structured question in the tool_call args; sometimes a richer
|
||||
# follow-up AIMessage is also produced after the ToolMessage. Emit a
|
||||
# dedicated frame so the frontend can render it as a question instead of
|
||||
# treating an empty dispatch as "consensus reached".
|
||||
clarification_call = next(
|
||||
(tc for tc in tool_calls if (tc.get("name") or "").lower() == "ask_clarification"),
|
||||
None,
|
||||
)
|
||||
if clarification_call is not None:
|
||||
cargs = clarification_call.get("args") or {}
|
||||
question = str(cargs.get("question") or "").strip()
|
||||
follow_up = _find_trailing_ai_text(messages, exclude_text=leader_text)
|
||||
yield {
|
||||
"status": "clarification",
|
||||
"content": follow_up or question or leader_text,
|
||||
"question": question,
|
||||
"clarification_type": str(cargs.get("clarification_type") or ""),
|
||||
"clarification_context": str(cargs.get("context") or ""),
|
||||
"options": cargs.get("options") or [],
|
||||
"allow_custom": bool(cargs.get("allow_custom", True)),
|
||||
"agent_name": agent_name,
|
||||
}
|
||||
return
|
||||
|
||||
if not tool_calls:
|
||||
yield {"status": [], "content": leader_text, "agent_name": agent_name}
|
||||
return
|
||||
|
||||
# Run all dispatch broadcasts concurrently — they target disjoint thread
|
||||
# sets and we don't care about ordering, so serializing them just
|
||||
# multiplied latency before the frontend could start a sub-agent run.
|
||||
dispatched: list[list[str]] = []
|
||||
dispatch_broadcasts: list[asyncio.Task[None]] = []
|
||||
for tool_call in tool_calls:
|
||||
try:
|
||||
sub_name, task_text = _extract_dispatch(tool_call)
|
||||
if not sub_name:
|
||||
continue
|
||||
dispatch_broadcasts.append(
|
||||
asyncio.create_task(
|
||||
_broadcast(
|
||||
client,
|
||||
base,
|
||||
headers,
|
||||
f"总控智能体给{sub_name}派活,内容为:" + task_text,
|
||||
agent_threads,
|
||||
[agent_name, sub_name],
|
||||
)
|
||||
)
|
||||
)
|
||||
dispatched.append([sub_name, task_text])
|
||||
except Exception as exc:
|
||||
logger.warning("dispatch failed: %s", exc)
|
||||
if dispatch_broadcasts:
|
||||
t_before_dispatch = time.perf_counter()
|
||||
await asyncio.gather(*dispatch_broadcasts, return_exceptions=True)
|
||||
logger.info(
|
||||
"[leader-timing] %s dispatch_broadcasts_await=%.3fs count=%d (gates how soon the next sub agent can start)",
|
||||
agent_name,
|
||||
time.perf_counter() - t_before_dispatch,
|
||||
len(dispatch_broadcasts),
|
||||
)
|
||||
yield {"status": dispatched, "content": leader_text, "agent_name": agent_name}
|
||||
|
||||
|
||||
async def _special_run(
|
||||
client: httpx.AsyncClient,
|
||||
base: str,
|
||||
headers: dict[str, str],
|
||||
agent_threads: dict[str, str],
|
||||
agent_name: str,
|
||||
new_message: str,
|
||||
skill_stop_names: list[str],
|
||||
model_name: str,
|
||||
) -> AsyncIterator[str | dict[str, Any]]:
|
||||
# Same instrumentation as _leader_run so all 6 seats are diagnosable.
|
||||
t0 = time.perf_counter()
|
||||
thread_id = agent_threads[agent_name]
|
||||
body = _run_payload(
|
||||
agent_name=agent_name,
|
||||
model_name=model_name,
|
||||
thread_id=thread_id,
|
||||
new_message=new_message,
|
||||
excluded_tools=["web_search", "ask_clarification", "present_files", "view_image", "agent_orchestration"],
|
||||
skill_stop_names=skill_stop_names,
|
||||
)
|
||||
t_payload_built = time.perf_counter()
|
||||
logger.info(
|
||||
"[special-timing] %s payload_built=%.3fs",
|
||||
agent_name,
|
||||
t_payload_built - t0,
|
||||
)
|
||||
|
||||
captured: list[str] = []
|
||||
first_line_ts: float | None = None
|
||||
line_count = 0
|
||||
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
|
||||
if line is not None:
|
||||
if first_line_ts is None:
|
||||
first_line_ts = time.perf_counter()
|
||||
logger.info(
|
||||
"[special-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
|
||||
agent_name,
|
||||
first_line_ts - t_payload_built,
|
||||
first_line_ts - t0,
|
||||
)
|
||||
line_count += 1
|
||||
yield line
|
||||
elif done is not None:
|
||||
captured = done
|
||||
t_stream_done = time.perf_counter()
|
||||
logger.info(
|
||||
"[special-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
|
||||
agent_name,
|
||||
t_stream_done - (first_line_ts or t_payload_built),
|
||||
line_count,
|
||||
t_stream_done - t0,
|
||||
)
|
||||
|
||||
messages = _parse_last_messages(captured)
|
||||
# Sub-agent's final deliverable is the latest AIMessage's text content.
|
||||
last_msg: dict[str, Any] = {}
|
||||
for msg in reversed(messages):
|
||||
if isinstance(msg, dict) and (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
|
||||
last_msg = msg
|
||||
break
|
||||
content = _flatten_content(last_msg.get("content", ""))
|
||||
# Kick the "子智能体完成" broadcast off in the background and yield the
|
||||
# status frame first — the next leader cycle that depends on this
|
||||
# broadcast having landed only starts after a fresh /run/stream request
|
||||
# from the frontend, which is many seconds away (full leader LLM call).
|
||||
# Awaiting after the yield ensures the task finishes before the AsyncClient
|
||||
# closes; the SSE connection stays open a bit longer but the frontend has
|
||||
# already consumed the result.
|
||||
t_before_post = time.perf_counter()
|
||||
post_broadcast_task = asyncio.create_task(
|
||||
_broadcast(
|
||||
client,
|
||||
base,
|
||||
headers,
|
||||
f"子智能体{agent_name}完成{new_message}工作,交付内容为:" + content,
|
||||
agent_threads,
|
||||
[agent_name],
|
||||
)
|
||||
)
|
||||
yield {
|
||||
"status": f"子智能体{agent_name}完成{new_message}工作",
|
||||
"content": content,
|
||||
"agent_name": agent_name,
|
||||
}
|
||||
try:
|
||||
await post_broadcast_task
|
||||
except Exception as exc:
|
||||
logger.warning("post-broadcast failed: %s", exc)
|
||||
logger.info(
|
||||
"[special-timing] %s post_broadcast_total=%.3fs (broadcast time after yield — keeps SSE open this long)",
|
||||
agent_name,
|
||||
time.perf_counter() - t_before_post,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/run/stream")
|
||||
async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) -> StreamingResponse:
|
||||
agent_type: str = str(payload.get("agent_type") or "leader").lower()
|
||||
agent_threads: dict[str, str] = payload.get("d_agent_thread_id") or {}
|
||||
agent_name: str = str(payload.get("agent_name") or "").lower()
|
||||
new_message: str = str(payload.get("new_message") or "")
|
||||
skill_stop_names: list[str] = payload.get("skill_stop_names") or []
|
||||
model_name: str = str(payload.get("model") or "deepseek-chat")
|
||||
|
||||
if not agent_name or agent_name not in agent_threads:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"agent_name '{agent_name}' is missing from d_agent_thread_id",
|
||||
)
|
||||
_validate_agent_id(agent_name, "agent_name")
|
||||
for key in agent_threads:
|
||||
_validate_agent_id(key, "d_agent_thread_id key")
|
||||
|
||||
base = _loopback_base(request)
|
||||
headers = _auth_headers(request)
|
||||
|
||||
async def stream_generator() -> AsyncIterator[str]:
|
||||
async with httpx.AsyncClient(timeout=None) as client:
|
||||
try:
|
||||
if agent_type == "leader":
|
||||
gen = _leader_run(
|
||||
client, base, headers, agent_threads, agent_name, new_message, skill_stop_names, model_name,
|
||||
)
|
||||
else:
|
||||
gen = _special_run(
|
||||
client, base, headers, agent_threads, agent_name, new_message, skill_stop_names, model_name,
|
||||
)
|
||||
async for chunk in gen:
|
||||
if isinstance(chunk, dict):
|
||||
yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"
|
||||
else:
|
||||
yield chunk if chunk.endswith("\n") else chunk + "\n"
|
||||
except HTTPException as exc:
|
||||
logger.warning("multi-agent stream HTTPException: %s %s", exc.status_code, exc.detail)
|
||||
err_frame = {
|
||||
"status": "error",
|
||||
"content": f"[{exc.status_code}] {exc.detail}",
|
||||
"agent_name": agent_name,
|
||||
}
|
||||
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
|
||||
except Exception as exc:
|
||||
logger.exception("multi-agent stream unexpected failure")
|
||||
err_frame = {
|
||||
"status": "error",
|
||||
"content": f"unexpected: {exc!r}",
|
||||
"agent_name": agent_name,
|
||||
}
|
||||
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
stream_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Public (unauthenticated) API for scheduled tasks.
|
||||
|
||||
Mounted under the ``/api/public/`` prefix so the auth middleware lets these
|
||||
routes through without a session cookie. Intended for embedding scheduled
|
||||
task results into external dashboards / documentation / chat systems that
|
||||
shouldn't need a DeerFlow login.
|
||||
|
||||
Surface area:
|
||||
|
||||
- ``GET /api/public/scheduled-tasks/`` — list every task (no prompt body)
|
||||
- ``GET /api/public/scheduled-tasks/{task_id}`` — task metadata
|
||||
- ``GET /api/public/scheduled-tasks/{task_id}/runs`` — runs for a task (latest first)
|
||||
- ``GET /api/public/scheduled-tasks/runs/{run_id}`` — single run (result included)
|
||||
- ``GET /api/public/scheduled-tasks/runs/{run_id}/files/{i}``— inline file content / preview
|
||||
- ``PUT /api/public/scheduled-tasks/runs/{run_id}/files/{i}``— overwrite an editable result file
|
||||
- ``PUT /api/public/scheduled-tasks/runs/{run_id}/body`` — overwrite the inline Markdown body
|
||||
- ``GET /api/public/scheduled-tasks/runs/{run_id}/download`` — zip / md download of result
|
||||
- ``DELETE /api/public/scheduled-tasks/runs/{run_id}`` — remove a single run (no auth)
|
||||
|
||||
The PUT endpoints intentionally skip ownership checks so the anonymous public
|
||||
viewer at ``#/public/scheduled-tasks/<task_id>`` can fix typos in result MD
|
||||
files / body content. Tighten this if the deployment requires read-only
|
||||
sharing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from fastapi.responses import FileResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.runtime.scheduler import get_scheduled_task_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Whitelisted under ``auth_middleware._PUBLIC_PATH_PREFIXES`` (``/api/public/``).
|
||||
router = APIRouter(prefix="/api/public/scheduled-tasks", tags=["public-scheduled-tasks"])
|
||||
|
||||
|
||||
_MAX_PUBLIC_EDIT_BYTES = 5 * 1024 * 1024 # 5 MB — mirrors the authenticated edit cap
|
||||
|
||||
|
||||
class PublicRunFileUpdateRequest(BaseModel):
|
||||
content: str = Field(..., description="New text content to write into the run file")
|
||||
|
||||
|
||||
class PublicRunBodyUpdateRequest(BaseModel):
|
||||
content: str = Field(..., description="New Markdown body for the run's result")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sanitisation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Fields stripped from every task before it leaves the public boundary.
|
||||
# Keeps owner ids, raw cron, and the (often verbose) author prompt out of
|
||||
# anonymous responses.
|
||||
_TASK_PRIVATE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"user_id",
|
||||
"owner_user_id",
|
||||
"creator_user_id",
|
||||
"creator_email",
|
||||
"prompt",
|
||||
"execution_context",
|
||||
"_subscription",
|
||||
"role",
|
||||
"scheduler_thread_id",
|
||||
}
|
||||
)
|
||||
|
||||
# Fields stripped from every run before it leaves the public boundary.
|
||||
# We keep the result body (the user explicitly wants the MD content surfaced)
|
||||
# but redact any path that points at a server-side filesystem location.
|
||||
_RUN_PRIVATE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"user_id",
|
||||
"owner_user_id",
|
||||
"scheduler_thread_id",
|
||||
"agent_run_id",
|
||||
"can_edit",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _redact_task(task: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in task.items() if k not in _TASK_PRIVATE_FIELDS}
|
||||
|
||||
|
||||
def _redact_run(run: dict[str, Any]) -> dict[str, Any]:
|
||||
cleaned = {k: v for k, v in run.items() if k not in _RUN_PRIVATE_FIELDS}
|
||||
result = cleaned.get("result")
|
||||
if isinstance(result, dict):
|
||||
files = result.get("files")
|
||||
if isinstance(files, list):
|
||||
cleaned_files = []
|
||||
for idx, item in enumerate(files):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
# Strip absolute filesystem path; clients use the file index
|
||||
# to fetch content via the public file endpoint.
|
||||
public_file = {k: v for k, v in item.items() if k != "path"}
|
||||
public_file["index"] = idx
|
||||
# Mark whether the underlying archive is still on disk. The
|
||||
# public viewer filters out tabs whose source is gone so users
|
||||
# never see the raw "Result file is no longer available" 404.
|
||||
path_value = item.get("path")
|
||||
public_file["available"] = (
|
||||
isinstance(path_value, str)
|
||||
and bool(path_value)
|
||||
and Path(path_value).is_file()
|
||||
)
|
||||
cleaned_files.append(public_file)
|
||||
result = {**result, "files": cleaned_files}
|
||||
cleaned["result"] = result
|
||||
return cleaned
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _safe_filename(value: str, default: str = "scheduled-task-result") -> str:
|
||||
name = re.sub(r'[\\/:*?"<>|\r\n]+', "-", value).strip(" .")
|
||||
return name or default
|
||||
|
||||
|
||||
def _result_markdown(result: dict[str, Any]) -> str:
|
||||
title = str(result.get("title") or "Scheduled Task Result")
|
||||
content = str(result.get("content") or "")
|
||||
return f"# {title}\n\n{content}".strip() + "\n"
|
||||
|
||||
|
||||
def _utf8_sig_bytes(content: str) -> bytes:
|
||||
return content.encode("utf-8-sig")
|
||||
|
||||
|
||||
def _content_disposition(filename: str) -> str:
|
||||
quoted = quote(filename)
|
||||
return f"attachment; filename*=UTF-8''{quoted}"
|
||||
|
||||
|
||||
def _run_file_path(run: dict[str, Any], file_index: int) -> tuple[dict[str, Any], Path]:
|
||||
result = run.get("result") or {}
|
||||
files = result.get("files") if isinstance(result, dict) and isinstance(result.get("files"), list) else []
|
||||
if file_index < 0 or file_index >= len(files):
|
||||
raise HTTPException(status_code=404, detail="Result file not found")
|
||||
item = files[file_index]
|
||||
if not isinstance(item, dict) or not isinstance(item.get("path"), str):
|
||||
raise HTTPException(status_code=404, detail="Result file not found")
|
||||
path = Path(item["path"])
|
||||
if not path.exists() or not path.is_file():
|
||||
raise HTTPException(status_code=404, detail="Result file is no longer available")
|
||||
return item, path
|
||||
|
||||
|
||||
def _is_text_previewable(path: Path, mime_type: str | None) -> bool:
|
||||
if mime_type and (mime_type.startswith("text/") or mime_type in {"application/json", "application/xml"}):
|
||||
return True
|
||||
return path.suffix.lower() in {".md", ".markdown", ".txt", ".json", ".csv", ".xml", ".yaml", ".yml", ".log"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def public_list_tasks(
|
||||
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on task name"),
|
||||
limit: int = Query(default=200, ge=1, le=500),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
"""Return every scheduled task across all users.
|
||||
|
||||
Sensitive fields (user_id, raw prompt, scheduler thread id, …) are stripped
|
||||
before serialisation. The response shape is intentionally stable so external
|
||||
integrations can pin to it without surprises.
|
||||
"""
|
||||
service = get_scheduled_task_service()
|
||||
tasks = await service.admin_list_tasks()
|
||||
|
||||
if search:
|
||||
needle = search.strip().lower()
|
||||
tasks = [t for t in tasks if needle in str(t.get("name", "")).lower()]
|
||||
|
||||
total = len(tasks)
|
||||
sliced = tasks[offset : offset + limit]
|
||||
return {
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"tasks": [_redact_task(task) for task in sliced],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{task_id}")
|
||||
async def public_get_task(task_id: str) -> dict[str, Any]:
|
||||
"""Return metadata for a single task (without prompt body)."""
|
||||
service = get_scheduled_task_service()
|
||||
task = await service.store.get_task_any(task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
|
||||
return _redact_task(task)
|
||||
|
||||
|
||||
@router.get("/{task_id}/runs")
|
||||
async def public_list_task_runs(
|
||||
task_id: str,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
"""Return runs for a task, latest first."""
|
||||
service = get_scheduled_task_service()
|
||||
task = await service.store.get_task_any(task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
|
||||
runs = await service.admin_list_runs(task_id, limit=limit)
|
||||
return {
|
||||
"task": _redact_task(task),
|
||||
"total": len(runs),
|
||||
"runs": [_redact_run(run) for run in runs],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}")
|
||||
async def public_get_run(run_id: str) -> dict[str, Any]:
|
||||
"""Return a single run with its full result body (Markdown included)."""
|
||||
service = get_scheduled_task_service()
|
||||
run = await service.admin_get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return _redact_run(run)
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/files/{file_index}")
|
||||
async def public_get_run_file(run_id: str, file_index: int) -> dict[str, Any]:
|
||||
"""Return a single result-file entry with its inline text content where
|
||||
previewable. Binary files come back with ``previewable: false`` and the
|
||||
consumer should fetch the download endpoint instead."""
|
||||
service = get_scheduled_task_service()
|
||||
run = await service.admin_get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
item, path = _run_file_path(run, file_index)
|
||||
mime_type = item.get("mime_type") or mimetypes.guess_type(path.name)[0]
|
||||
previewable = _is_text_previewable(path, mime_type)
|
||||
content: str | None = None
|
||||
if previewable:
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
return {
|
||||
"index": file_index,
|
||||
"name": item.get("name") or path.name,
|
||||
"mime_type": mime_type,
|
||||
"kind": item.get("kind"),
|
||||
"size": path.stat().st_size,
|
||||
"previewable": previewable,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/files/{file_index}/download")
|
||||
async def public_download_run_file(run_id: str, file_index: int):
|
||||
"""Stream a single result file as a download."""
|
||||
service = get_scheduled_task_service()
|
||||
run = await service.admin_get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
item, path = _run_file_path(run, file_index)
|
||||
filename = _safe_filename(str(item.get("name") or path.name), path.name)
|
||||
media = item.get("mime_type") or mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||
return FileResponse(path, filename=filename, media_type=media)
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/download")
|
||||
async def public_download_run(run_id: str):
|
||||
"""Download the full result: a zipped bundle of result.md + every file,
|
||||
or a single ``.md`` when no auxiliary files were produced."""
|
||||
service = get_scheduled_task_service()
|
||||
run = await service.admin_get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
|
||||
result = run.get("result") or {}
|
||||
if not isinstance(result, dict):
|
||||
result = {}
|
||||
files = result.get("files") if isinstance(result.get("files"), list) else []
|
||||
accessible_files: list[tuple[str, Path]] = []
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
path_value = item.get("path")
|
||||
if not isinstance(path_value, str) or not path_value:
|
||||
continue
|
||||
path = Path(path_value)
|
||||
if path.exists() and path.is_file():
|
||||
accessible_files.append((str(item.get("name") or path.name), path))
|
||||
|
||||
base_name = _safe_filename(str(result.get("title") or "scheduled-task-result"))
|
||||
if not accessible_files:
|
||||
return Response(
|
||||
content=_utf8_sig_bytes(_result_markdown(result)),
|
||||
media_type="text/markdown; charset=utf-8",
|
||||
headers={"Content-Disposition": _content_disposition(f"{base_name}.md")},
|
||||
)
|
||||
|
||||
archive = BytesIO()
|
||||
with zipfile.ZipFile(archive, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr("result.md", _utf8_sig_bytes(_result_markdown(result)))
|
||||
for name, path in accessible_files:
|
||||
arcname = _safe_filename(name, path.name)
|
||||
zf.write(path, arcname=arcname)
|
||||
archive.seek(0)
|
||||
return StreamingResponse(
|
||||
archive,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": _content_disposition(f"{base_name}.zip")},
|
||||
)
|
||||
|
||||
|
||||
@router.put("/runs/{run_id}/files/{file_index}")
|
||||
async def public_update_run_file(
|
||||
run_id: str,
|
||||
file_index: int,
|
||||
body: PublicRunFileUpdateRequest,
|
||||
) -> dict[str, Any]:
|
||||
"""Overwrite an MD/text result file on disk and refresh its size in the run row.
|
||||
|
||||
Anonymous — no ownership check. See module docstring for the rationale.
|
||||
"""
|
||||
if len(body.content.encode("utf-8")) > _MAX_PUBLIC_EDIT_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Edited content exceeds the 5 MB limit")
|
||||
service = get_scheduled_task_service()
|
||||
try:
|
||||
return await service.admin_update_run_file(run_id, file_index, body.content)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
except PermissionError as exc:
|
||||
# Surfaces archive-path escape attempts. Treat as 400 since the caller is
|
||||
# already trusted to be reaching a real run row.
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
|
||||
|
||||
@router.delete("/runs/{run_id}", status_code=204)
|
||||
async def public_delete_run(run_id: str) -> Response:
|
||||
"""Remove a single run regardless of owner.
|
||||
|
||||
Anonymous — no ownership check, mirroring the existing public PUT endpoints.
|
||||
Tighten this if the deployment requires read-only public sharing.
|
||||
"""
|
||||
service = get_scheduled_task_service()
|
||||
deleted = await service.admin_delete_run(run_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.put("/runs/{run_id}/body")
|
||||
async def public_update_run_body(run_id: str, body: PublicRunBodyUpdateRequest) -> dict[str, Any]:
|
||||
"""Overwrite the inline Markdown body (``result.content``) of a run.
|
||||
|
||||
Anonymous — no ownership check. Used by the public viewer's "正文" tab.
|
||||
"""
|
||||
if len(body.content.encode("utf-8")) > _MAX_PUBLIC_EDIT_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Edited content exceeds the 5 MB limit")
|
||||
service = get_scheduled_task_service()
|
||||
try:
|
||||
return await service.admin_update_run_body(run_id, body.content)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
@@ -14,7 +14,16 @@ from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_local_provider, get_optional_user_from_request, get_run_event_store, get_run_store, get_tag_store
|
||||
from app.gateway.deps import (
|
||||
get_checkpointer,
|
||||
get_current_user,
|
||||
get_local_provider,
|
||||
get_optional_user_from_request,
|
||||
get_run_event_store,
|
||||
get_run_store,
|
||||
get_tag_store,
|
||||
)
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.scheduler import DuplicateTaskNameError, get_scheduled_task_service
|
||||
|
||||
router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
|
||||
@@ -22,6 +31,7 @@ router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
|
||||
|
||||
class ScheduledTaskCreateRequest(BaseModel):
|
||||
name: str = Field(default="Scheduled task")
|
||||
description: str | None = None
|
||||
prompt: str
|
||||
schedule: str
|
||||
timezone: str | None = None
|
||||
@@ -31,6 +41,7 @@ class ScheduledTaskCreateRequest(BaseModel):
|
||||
|
||||
class ScheduledTaskUpdateRequest(BaseModel):
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
prompt: str | None = None
|
||||
schedule: str | None = None
|
||||
timezone: str | None = None
|
||||
@@ -235,31 +246,129 @@ async def get_scheduled_task_run(run_id: str, request: Request) -> dict:
|
||||
return run
|
||||
|
||||
|
||||
def _event_type_for_checkpoint_message(message: dict) -> str:
|
||||
"""Map a serialized LangChain message to the same ``event_type`` strings
|
||||
the run-event store uses, so the frontend's existing role labels keep
|
||||
working for checkpoint-derived synthetic rows."""
|
||||
raw_type = message.get("type") or message.get("role") or ""
|
||||
if raw_type in {"human", "user"}:
|
||||
return "llm.human.input"
|
||||
if raw_type in {"ai", "assistant"}:
|
||||
return "llm.ai.response"
|
||||
if raw_type == "tool":
|
||||
return "llm.tool.result"
|
||||
if raw_type == "system":
|
||||
return "llm.system"
|
||||
return f"llm.{raw_type or 'message'}"
|
||||
|
||||
|
||||
async def _checkpoint_messages_for_run(
|
||||
request: Request, agent_run: dict, *, limit: int
|
||||
) -> list[dict]:
|
||||
"""Return a synthetic list of message rows pulled from the LangGraph
|
||||
checkpoint for an in-flight run.
|
||||
|
||||
The event store is updated by the streaming pipeline and may stay empty
|
||||
until the first message tuple is flushed — meanwhile the checkpoint
|
||||
already carries the current state (input + accumulating AI deltas). We
|
||||
project that into the same row shape the frontend uses for the event
|
||||
store, so the run-detail page can show progress immediately."""
|
||||
try:
|
||||
checkpointer = get_checkpointer(request)
|
||||
except Exception:
|
||||
return []
|
||||
thread_id = agent_run.get("thread_id")
|
||||
if not thread_id:
|
||||
return []
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
try:
|
||||
tup = await checkpointer.aget_tuple(config)
|
||||
except Exception:
|
||||
return []
|
||||
if tup is None:
|
||||
return []
|
||||
checkpoint = getattr(tup, "checkpoint", {}) or {}
|
||||
channel_values = checkpoint.get("channel_values") or {}
|
||||
try:
|
||||
values = serialize_channel_values(channel_values)
|
||||
except Exception:
|
||||
return []
|
||||
messages = values.get("messages") or []
|
||||
|
||||
rows: list[dict] = []
|
||||
for index, message in enumerate(messages[-limit:]):
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"seq": index + 1,
|
||||
"event_type": _event_type_for_checkpoint_message(message),
|
||||
"category": "message",
|
||||
"content": message,
|
||||
"metadata": {"source": "checkpoint"},
|
||||
"created_at": None,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/messages")
|
||||
async def get_scheduled_task_run_messages(run_id: str, request: Request, limit: int = 500) -> dict:
|
||||
"""Agent messages produced by a scheduled run's underlying agent run.
|
||||
|
||||
Lets the run-detail page show what the agent is doing — useful when a run
|
||||
is stuck in ``running``. Authorization reuses :func:`_load_run` (owner /
|
||||
subscriber / admin); the event-store query then passes ``user_id=None``
|
||||
because access has already been checked.
|
||||
Resolution order (most authoritative first):
|
||||
|
||||
1. Event-store rows for the underlying agent run — these are the per-tuple
|
||||
streaming events the worker writes as the agent runs.
|
||||
2. LangGraph checkpoint snapshot — the canonical conversation state. Used
|
||||
as a fallback when the event store is still empty (early in a run) or
|
||||
when only partial events have been flushed; we project the snapshot
|
||||
into the same row shape the frontend already renders so the user can
|
||||
see the prompt + current AI output while the task is mid-flight.
|
||||
3. Synthetic human-input row built from ``task.prompt`` — shown even
|
||||
before the agent run is registered, so a `pending` task at minimum
|
||||
displays what was queued.
|
||||
"""
|
||||
service = get_scheduled_task_service()
|
||||
user_id = await _user_id(request)
|
||||
run = await _load_run(service, request, run_id, user_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
capped = max(1, min(limit, 1000))
|
||||
|
||||
# Always available: task prompt as the seeded human input. Lets the
|
||||
# ``execution process`` tab show *something* even before the agent run is
|
||||
# registered, which is the user-visible bug we're fixing.
|
||||
fallback_rows: list[dict] = []
|
||||
task_id = run.get("task_id")
|
||||
if task_id:
|
||||
task = await service.store.get_task_any(task_id)
|
||||
prompt = (task or {}).get("prompt")
|
||||
if prompt:
|
||||
fallback_rows.append(
|
||||
{
|
||||
"seq": 0,
|
||||
"event_type": "llm.human.input",
|
||||
"category": "message",
|
||||
# Frontend treats human-input content as a plain string.
|
||||
"content": str(prompt),
|
||||
"metadata": {"source": "task_prompt"},
|
||||
"created_at": run.get("created_at") or run.get("scheduled_for"),
|
||||
}
|
||||
)
|
||||
|
||||
agent_run_id = run.get("agent_run_id")
|
||||
if not agent_run_id:
|
||||
return {"messages": [], "has_more": False}
|
||||
return {"messages": fallback_rows, "has_more": False}
|
||||
|
||||
# Each scheduled run executes on its own throwaway runtime thread — NOT the
|
||||
# run row's ``scheduler_thread_id`` — so resolve the real thread from the
|
||||
# run store before querying its message events.
|
||||
run_store = get_run_store(request)
|
||||
agent_run = await run_store.get(agent_run_id, user_id=None)
|
||||
if agent_run is None:
|
||||
return {"messages": [], "has_more": False}
|
||||
capped = max(1, min(limit, 1000))
|
||||
return {"messages": fallback_rows, "has_more": False}
|
||||
|
||||
event_store = get_run_event_store(request)
|
||||
rows = await event_store.list_messages_by_run(
|
||||
agent_run["thread_id"],
|
||||
@@ -267,6 +376,16 @@ async def get_scheduled_task_run_messages(run_id: str, request: Request, limit:
|
||||
limit=capped + 1,
|
||||
user_id=None,
|
||||
)
|
||||
|
||||
# Event store has no rows yet (run just started, or events still buffering).
|
||||
# Fall back to the live checkpoint so the user immediately sees the prompt
|
||||
# + whatever AI output has been streamed so far.
|
||||
if not rows:
|
||||
checkpoint_rows = await _checkpoint_messages_for_run(request, agent_run, limit=capped)
|
||||
if checkpoint_rows:
|
||||
return {"messages": checkpoint_rows[:capped], "has_more": False}
|
||||
return {"messages": fallback_rows, "has_more": False}
|
||||
|
||||
has_more = len(rows) > capped
|
||||
return {"messages": rows[:capped] if has_more else rows, "has_more": has_more}
|
||||
|
||||
@@ -430,6 +549,7 @@ async def create_scheduled_task(body: ScheduledTaskCreateRequest, request: Reque
|
||||
task = await service.create_task(
|
||||
user_id=user_id,
|
||||
name=body.name,
|
||||
description=body.description,
|
||||
prompt=body.prompt,
|
||||
schedule=body.schedule,
|
||||
timezone=body.timezone,
|
||||
|
||||
@@ -161,6 +161,13 @@ class SkillUploadValidateResponse(BaseModel):
|
||||
warnings: list[str] = Field(default_factory=list, description="Non-fatal warnings to surface to the user")
|
||||
|
||||
|
||||
class SkillReconcileResponse(BaseModel):
|
||||
"""Response model for reconciling custom skills found on disk."""
|
||||
|
||||
adopted: list[str] = Field(default_factory=list, description="Skills found on disk that had no DB row; now owned by the caller")
|
||||
already_registered: list[str] = Field(default_factory=list, description="Skills found on disk that already had a DB row (unchanged)")
|
||||
|
||||
|
||||
class CustomSkillContentResponse(SkillResponse):
|
||||
content: str = Field(..., description="Raw SKILL.md content")
|
||||
|
||||
@@ -483,6 +490,54 @@ async def install_skill(
|
||||
raise HTTPException(status_code=500, detail=f"Failed to install skill: {str(e)}")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/skills/reconcile-custom",
|
||||
response_model=SkillReconcileResponse,
|
||||
summary="Adopt Orphan Custom Skills",
|
||||
description=(
|
||||
"Scan the on-disk ``skills/custom/`` directory and register any skill that has a "
|
||||
"valid SKILL.md but no DB ownership row. Adopted skills are recorded as owned by "
|
||||
"the calling user with ``published=false``. Skips skills that already have a row "
|
||||
"(regardless of owner). Useful after manually unzipping skills onto the filesystem."
|
||||
),
|
||||
)
|
||||
async def reconcile_custom_skills(
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
) -> SkillReconcileResponse:
|
||||
try:
|
||||
user_id = _current_user_id(request)
|
||||
storage = get_or_new_skill_storage(app_config=config)
|
||||
custom_skills = [s for s in storage.load_skills(enabled_only=False) if s.category == SkillCategory.CUSTOM]
|
||||
|
||||
adopted: list[str] = []
|
||||
already: list[str] = []
|
||||
for skill in custom_skills:
|
||||
existing = await skill_store.get_any(skill.name)
|
||||
if existing is not None:
|
||||
already.append(skill.name)
|
||||
continue
|
||||
await skill_store.ensure_legacy(
|
||||
{"name": skill.name, "owner_user_id": user_id, "published": False}
|
||||
)
|
||||
try:
|
||||
from deerflow.skills.usage import mark_uploaded
|
||||
mark_uploaded(skill.name)
|
||||
except Exception:
|
||||
pass
|
||||
adopted.append(skill.name)
|
||||
|
||||
if adopted:
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return SkillReconcileResponse(adopted=adopted, already_registered=already)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to reconcile custom skills: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to reconcile custom skills: {e}")
|
||||
|
||||
|
||||
@router.get("/skills/custom", response_model=SkillsListResponse, summary="List Custom Skills")
|
||||
async def list_custom_skills(
|
||||
request: Request,
|
||||
|
||||
@@ -54,6 +54,9 @@ class RunCreateRequest(BaseModel):
|
||||
after_seconds: float | None = Field(default=None, description="Delayed execution")
|
||||
if_not_exists: Literal["reject", "create"] = Field(default="create", description="Thread creation policy")
|
||||
feedback_keys: list[str] | None = Field(default=None, description="LangSmith feedback keys")
|
||||
excluded_tools: list[str] | None = Field(default=None, description="Tool names to exclude from the main agent (e.g. ['task', 'ask_clarification'])")
|
||||
skill_stop_names: list[str] | None = Field(default=None, description="Skill names to stop execution when invoked (e.g. ['newsletter-generation'])")
|
||||
skill_stop_message: str | None = Field(default=None, description="Custom message to display when skill stop is triggered")
|
||||
|
||||
|
||||
class RunResponse(BaseModel):
|
||||
|
||||
@@ -100,7 +100,7 @@ _DEFAULT_ASSISTANT_ID = "lead_agent"
|
||||
|
||||
# Built-in LangGraph graph IDs registered in langgraph.json.
|
||||
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
|
||||
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
|
||||
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent", "ai_writing"})
|
||||
|
||||
|
||||
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
|
||||
@@ -162,6 +162,9 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
"max_concurrent_subagents",
|
||||
"agent_name",
|
||||
"is_bootstrap",
|
||||
"excluded_tools",
|
||||
"skill_stop_names",
|
||||
"skill_stop_message",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -186,15 +189,20 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
|
||||
def resolve_agent_factory(assistant_id: str | None):
|
||||
"""Resolve the agent factory callable from config.
|
||||
|
||||
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
|
||||
to their own factory. Custom agents share the ``lead_agent`` factory with
|
||||
``agent_id`` injected into configurable/context by :func:`build_run_config`.
|
||||
Built-in graphs (``lead_agent``, ``canvas_agent``, ``ai_writing``) are dispatched
|
||||
directly to their own factory. Custom agents share the ``lead_agent`` factory
|
||||
with ``agent_id`` injected into configurable/context by :func:`build_run_config`.
|
||||
"""
|
||||
if assistant_id == "canvas_agent":
|
||||
from deerflow.agents.canvas_agent.agent import make_canvas_agent
|
||||
|
||||
return make_canvas_agent
|
||||
|
||||
if assistant_id == "ai_writing":
|
||||
from deerflow.agents.ai_writing.graph import make_ai_writing_graph
|
||||
|
||||
return make_ai_writing_graph
|
||||
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
|
||||
return make_lead_agent
|
||||
@@ -400,6 +408,11 @@ async def start_run(
|
||||
)
|
||||
else:
|
||||
await run_ctx.thread_store.update_status(thread_id, "running")
|
||||
# SDK 的 ``client.threads.create()`` 不传 assistant_id —— 先建出来空的 thread,
|
||||
# 真正提交 run 时才知道用哪个 assistant。这里回填一次,避免历史列表 / 按
|
||||
# assistant 筛选时拿不到关联。
|
||||
if not existing.get("assistant_id") and body.assistant_id:
|
||||
await run_ctx.thread_store.update_assistant_id(thread_id, body.assistant_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
|
||||
|
||||
@@ -424,7 +437,15 @@ async def start_run(
|
||||
# The ``context`` field is a custom extension for the langgraph-compat layer
|
||||
# that carries agent configuration (model_name, thinking_enabled, etc.).
|
||||
# Only agent-relevant keys are forwarded; unknown keys (e.g. thread_id) are ignored.
|
||||
merge_run_context_overrides(config, getattr(body, "context", None))
|
||||
body_context = dict(getattr(body, "context", None) or {})
|
||||
# Promote whitelisted top-level body fields into the context map so
|
||||
# callers can pass them either nested under ``context`` or as flat
|
||||
# top-level fields (``excluded_tools``, ``skill_stop_names``, …).
|
||||
for top_level_key in ("excluded_tools", "skill_stop_names", "skill_stop_message"):
|
||||
value = getattr(body, top_level_key, None)
|
||||
if value is not None and top_level_key not in body_context:
|
||||
body_context[top_level_key] = value
|
||||
merge_run_context_overrides(config, body_context or None)
|
||||
|
||||
stream_modes = normalize_stream_modes(body.stream_mode)
|
||||
|
||||
@@ -441,6 +462,9 @@ async def start_run(
|
||||
stream_subgraphs=body.stream_subgraphs,
|
||||
interrupt_before=body.interrupt_before,
|
||||
interrupt_after=body.interrupt_after,
|
||||
# 透传 SDK ``Command``:resume / update / goto。Pregel 会据此从最近
|
||||
# checkpoint 处恢复,而不是把 graph_input 当新一轮输入跑。
|
||||
command=getattr(body, "command", None),
|
||||
)
|
||||
)
|
||||
record.task = task
|
||||
|
||||
@@ -41,6 +41,26 @@ models:
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: MiniMax-M2.7
|
||||
display_name: MiniMax-M2.7
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.7
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com/v1
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: minimax-m2.5
|
||||
display_name: MiniMax M2.5
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.5
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com/v1
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: false
|
||||
|
||||
title:
|
||||
enabled: true
|
||||
@@ -105,17 +125,40 @@ sandbox:
|
||||
|
||||
skills:
|
||||
path: D:\project\bj\deerflow-server\skills
|
||||
security_scan:
|
||||
enabled: false
|
||||
|
||||
agents_api:
|
||||
enabled: true
|
||||
|
||||
database:
|
||||
backend: mysql
|
||||
mysql_url: $MYSQL_DATABASE_URL
|
||||
pool_size: 10
|
||||
echo_sql: false
|
||||
|
||||
checkpointer:
|
||||
type: sqlite
|
||||
connection_string: D:\cursor-work\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
|
||||
|
||||
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
|
||||
# 三层覆盖优先级(高->低):
|
||||
# users/{user_id}/memory_config.json > .deer-flow/memory_user_overrides.json > 此处
|
||||
memory:
|
||||
enabled: true
|
||||
version: v1 # 记忆版本: 'v1'(原版 DeerFlow, memory.json + LLM 自动提取)
|
||||
# 'v2'(USER.md/MEMORY.md + 可选 Hindsight)
|
||||
provider: hindsight # 'builtin' | 'hindsight'(外部 provider 名)
|
||||
# builtin 始终启用,这里只决定 external 选哪个
|
||||
# (version=v1 时此字段忽略)
|
||||
|
||||
# V1 专属配置 —— version=v1 时生效
|
||||
v1:
|
||||
max_facts: 100 # 最多保留的事实条数,超出时驱逐最低置信度的事实
|
||||
fact_confidence_threshold: 0.7 # 事实置信度阈值,低于此值不保存
|
||||
debounce_seconds: 30 # 对话结束后延迟多少秒才触发 LLM 提取
|
||||
model_name: "" # 用于事实提取的 LLM 模型名;空则继承全局默认模型
|
||||
token_budget: 2000 # 注入系统提示词的 token 预算
|
||||
builtin:
|
||||
enabled: true
|
||||
memory_char_limit: 2200
|
||||
@@ -124,15 +167,15 @@ memory:
|
||||
|
||||
hindsight: # provider=hindsight 时生效,需另行部署 Hindsight 服务
|
||||
mode: local_external # 'cloud' | 'local_embedded' | 'local_external'
|
||||
api_url: http://localhost:8765
|
||||
api_url: http://47.88.25.99:18888
|
||||
api_key: "" # 部署 Hindsight 后填,或改成 $HINDSIGHT_API_KEY 并设同名环境变量
|
||||
# 注意:写成 $VAR 但环境变量未设置会导致配置加载失败
|
||||
bank_id_template: "deerflow-user-{user_id}"
|
||||
work_bank_id_template: "deerflow-work-{user_id}-{agent_id}"
|
||||
memory_mode: context # 'context' | 'tools' | 'hybrid'
|
||||
bank_mission: ""
|
||||
bank_retain_mission: ""
|
||||
recall_budget: mid # 'low' | 'mid' | 'high'
|
||||
memory_mode: context # 'context' | 'tools' | 'hybrid'
|
||||
bank_mission: "你是一个中文记忆助手,负责从对话中提炼和存储重要信息。所有记忆内容必须使用中文记录。"
|
||||
bank_retain_mission: "请从以下对话中提炼重要信息,包括用户的偏好、需求、背景知识和关键事实。所有提炼的内容必须用中文表达。"
|
||||
recall_budget: mid # 'low' | 'mid' | 'high'
|
||||
recall_max_tokens: 4096
|
||||
recall_max_input_chars: 800
|
||||
auto_recall: true
|
||||
@@ -158,10 +201,18 @@ memory:
|
||||
max_tokens: 2000
|
||||
include_builtin: true
|
||||
include_hindsight: true
|
||||
context_tag: "memory-context" # 包裹 Hindsight recall 的标签名
|
||||
context_tag: "memory-context" # 包裹 Hindsight recall 的标签名
|
||||
|
||||
security:
|
||||
scan_content: true # 注入/外渗模式扫描
|
||||
scan_content: true # 注入/外渗模式扫描
|
||||
block_invisible_unicode: true
|
||||
# streaming_scrubber:预留字段,代码尚未实现 —— 待版本支持后取消注释。
|
||||
# streaming_scrubber: true # SSE 流前清洗 <memory-context>
|
||||
|
||||
# AI 写作子系统 —— 素材收集专家专用配置。
|
||||
# 注意:这三项必须嵌套在 ai_writing: 段下;写到顶层会被 _ai_writing_config()
|
||||
# (app/gateway/routers/ai_writing.py:_ai_writing_config) 直接忽略。
|
||||
ai_writing:
|
||||
keyword_model: deepseek-chat # 检索词生成专用模型,留空回退写作主模型
|
||||
rank_model: deepseek-chat # 素材排序专用模型,留空回退写作主模型
|
||||
max_materials: 20 # 合并去重后的素材总量上限,范围 [1, 100],默认 20
|
||||
@@ -0,0 +1,845 @@
|
||||
# AI 写作 — 后端开发文档
|
||||
|
||||
> **本文档目的**:帮助后端开发者快速理解 AI 写作子系统的代码结构、Graph 拓扑、数据流、持久化、容错策略和扩展点。
|
||||
>
|
||||
> **配套文档**:
|
||||
> - 前端:`frontend-web/docs/ai-writing-frontend-dev.md`
|
||||
> - 故障排查:`frontend-web/docs/ai-writing-说明.md`
|
||||
> - 前端 API 字段索引:`frontend-web/docs/ai-writing-assistant-backend-api.md`
|
||||
|
||||
---
|
||||
|
||||
## 一、产品定位与技术架构
|
||||
|
||||
### 产品定位
|
||||
**杂志社协作写作子系统**:由 4 个 AI Agent(**素材收集专家 / 作家 / 编辑 / 系统**)协作完成一篇文章,通过 4 个用户干预点参与决策。
|
||||
|
||||
### 技术栈
|
||||
- **编排框架**:LangGraph(独立 StateGraph,不复用主 lead agent 的 middleware 链)
|
||||
- **节点驱动**:异步 LLM 调用(`langchain_openai` 等),支持流式 token 推送到 SSE
|
||||
- **状态持久化**:LangGraph checkpointer(SQLite 文件,跨重启可恢复)
|
||||
- **业务数据**:MySQL `ai_writing_sessions` 表
|
||||
- **协议**:SSE(Server-Sent Events)推进度 + REST 控制流
|
||||
|
||||
### 与主 Agent 的关系
|
||||
**完全独立**。AI 写作 graph 不通过 `make_lead_agent` 也不走主 agent 的 18 层 middleware。它有自己的 StateGraph、自己的 checkpointer、自己的路由前缀(`/api/ai-writing/*`),跟主 agent 解耦。
|
||||
|
||||
---
|
||||
|
||||
## 二、目录结构
|
||||
|
||||
```
|
||||
offline-backend-20260512/backend/
|
||||
├── app/gateway/
|
||||
│ ├── routers/
|
||||
│ │ └── ai_writing.py # FastAPI 路由 + 模块级状态字典 + SSE 生成器(1000+ 行,核心)
|
||||
│ ├── ai_writing_cleanup.py # 定期清理 + cron 调度(独立模块)
|
||||
│ └── app.py # lifespan 启动/停止清理任务、检测多 worker、关闭 checkpointer
|
||||
│
|
||||
└── packages/harness/deerflow/
|
||||
├── agents/ai_writing/
|
||||
│ ├── __init__.py # 暴露 AI_WRITING_GRAPH + 工厂函数
|
||||
│ ├── graph.py # ⭐ StateGraph 定义 + 4 个 pause 节点 + checkpointer 工厂(538 行)
|
||||
│ ├── state.py # AIWritingState TypedDict(121 行)
|
||||
│ │
|
||||
│ ├── nodes/ # ── Graph 节点 ──
|
||||
│ │ ├── _utils.py # LLM 工具:parse_json, ThinkTagParser, llm_json
|
||||
│ │ ├── intent_parser.py # 节点 1:意图解析
|
||||
│ │ ├── researcher.py # 节点 2:素材检索(支持流式 progress)
|
||||
│ │ ├── writer_outline.py # 节点 3:大纲规划
|
||||
│ │ ├── writer_draft.py # 节点 4:草稿生成(按章节流式)
|
||||
│ │ └── editor.py # 节点 5:编辑审核(3 维度并行流式)
|
||||
│ │
|
||||
│ ├── prompts/ # ── Prompt 模板 ──
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── researcher_prompts.py
|
||||
│ │ ├── writer_prompts.py
|
||||
│ │ └── editor_prompts.py
|
||||
│ │
|
||||
│ └── search/ # ── 素材检索 provider ──
|
||||
│ ├── base.py # SearchResult / SearchProvider 接口
|
||||
│ ├── duckduckgo_search.py
|
||||
│ ├── internet_search.py # 互联网检索(默认)
|
||||
│ ├── intranet_stub.py # 内网检索占位(按部署接入)
|
||||
│ └── demo_search.py # 离线 demo 数据
|
||||
│
|
||||
└── persistence/ai_writing_sessions/
|
||||
├── __init__.py # AIWritingSessionRepository 工厂
|
||||
├── model.py # ORM:AIWritingSessionRow
|
||||
└── sql.py # CRUD 实现(含清理用的 list_older_than / delete_by_ids)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、Graph 拓扑(最重要)
|
||||
|
||||
### StateGraph 节点 + 边
|
||||
|
||||
```
|
||||
START
|
||||
│
|
||||
↓
|
||||
┌─────────────────┐
|
||||
│ intent_parser │ 解析用户意图(确定 article_type / target_audience)
|
||||
└────────┬────────┘
|
||||
↓
|
||||
┌─────────────────┐
|
||||
│ researcher │ ← (re_search 回到这里)
|
||||
└────────┬────────┘
|
||||
↓ (after_researcher: ⼟材 ready)
|
||||
┌──────────────────────┐
|
||||
│ pause_material │ ⚙️ PAUSE-1:素材确认
|
||||
└─┬──────────┬─────────┘
|
||||
│ confirm │ re_search
|
||||
↓ └→ researcher
|
||||
┌─────────────────┐
|
||||
│ writer_outline │ ← (re_outline 回到这里)
|
||||
└────────┬────────┘
|
||||
↓
|
||||
┌──────────────────────┐
|
||||
│ pause_outline │ ⚙️ PAUSE-2:大纲确认
|
||||
└─┬──────────┬─────────┘
|
||||
│ confirm │ re_outline
|
||||
↓ └→ writer_outline
|
||||
┌─────────────────┐
|
||||
│ writer_draft │ ← (素材补充后回到这里)
|
||||
└─┬───────────────┘
|
||||
↓ (after_writer_draft)
|
||||
┌──────┴────────────────┐
|
||||
↓ ↓
|
||||
┌──────────────────┐ ┌────────────────────┐
|
||||
│ pause_section_ │ │ pause_draft │ ⚙️ PAUSE-3:草稿确认
|
||||
│ help(素材不足)│ │ │
|
||||
└─┬─────────────┬──┘ └─┬────────┬───┬─────┘
|
||||
│ supplement │ del/ │ │ │ to_editor
|
||||
│↓ │ loose │ │ user_revise
|
||||
research writer_ │ final ↓
|
||||
draft │ ize writer_draft
|
||||
│↓
|
||||
done
|
||||
│
|
||||
(to_editor)│
|
||||
↓
|
||||
┌─────────────────┐
|
||||
│ editor │ 3 维度并行流式审核
|
||||
└────────┬────────┘
|
||||
↓ (after_review)
|
||||
┌──────────────────────┐
|
||||
│ pause_review │ ⚙️ PAUSE-4:审核确认
|
||||
└─┬──────────┬─────────┘
|
||||
│ confirm │ revise
|
||||
│↓ └→ writer_draft
|
||||
done
|
||||
```
|
||||
|
||||
### 节点定义位置一览
|
||||
|
||||
| 节点名 | 实现文件 | 类型 | 流式? |
|
||||
|---|---|---|---|
|
||||
| `intent_parser` | `nodes/intent_parser.py` | async function | 否 |
|
||||
| `researcher` | `nodes/researcher.py` | async function | 是(research_progress) |
|
||||
| `pause_material` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
|
||||
| `writer_outline` | `nodes/writer_outline.py` | async function | 是(outline_chunk) |
|
||||
| `pause_outline` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
|
||||
| `writer_draft` | `nodes/writer_draft.py` | async function | 是(draft_chunk,按章节) |
|
||||
| `pause_section_help` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
|
||||
| `pause_draft` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
|
||||
| `editor` | `nodes/editor.py` | async function | 是(review_chunk,3 路并行) |
|
||||
| `pause_review` | `graph.py` 顶部 | async, 调 `interrupt()` | — |
|
||||
|
||||
### 条件路由函数(graph.py)
|
||||
|
||||
| 函数 | 在哪个节点之后 | 路由依据 |
|
||||
|---|---|---|
|
||||
| `after_researcher` | researcher | 是否找到素材 |
|
||||
| `after_material_pause` | pause_material | `state["status"]` 是 researching 还是 writing_outline |
|
||||
| `after_outline_pause` | pause_outline | 同上,writing_outline 还是 writing_draft |
|
||||
| `after_writer_draft` | writer_draft | 是否有 blocked_sections |
|
||||
| `after_section_help_pause` | pause_section_help | status |
|
||||
| `after_draft_pause` | pause_draft | done / revising / reviewing |
|
||||
| `after_review` | editor | review_result.verdict 是 pass 还是 reject + revision_count 上限 |
|
||||
| `after_review_pause` | pause_review | done / writing_draft |
|
||||
|
||||
---
|
||||
|
||||
## 四、State 定义(state.py)
|
||||
|
||||
```python
|
||||
class AIWritingState(TypedDict):
|
||||
# —— 写作配置(启动时填充)——
|
||||
user_intent: str
|
||||
article_type: str
|
||||
target_audience: str
|
||||
word_count_target: int
|
||||
keyword_count: int # 1-10,默认 6
|
||||
writing_mode: str # "strict" | "loose"
|
||||
|
||||
# —— Agent 产出 ——
|
||||
material_package: Optional[MaterialPackage] # 素材收集专家
|
||||
current_outline: Optional[Outline] # 作家
|
||||
drafts: Annotated[List[DraftArticle], operator.add] # 累计所有版本
|
||||
current_draft: Optional[DraftArticle] # 最新草稿
|
||||
review_result: Optional[ReviewResult] # 编辑
|
||||
|
||||
# —— 用户干预 ——
|
||||
last_user_intervention: Optional[UserInterventionPayload]
|
||||
blocked_sections: Optional[List[BlockedSection]] # 素材不足的章节
|
||||
|
||||
# —— 流程控制 ——
|
||||
revision_count: int
|
||||
max_revisions: int # 默认 3
|
||||
status: WritingStatus
|
||||
|
||||
# —— SSE 推送队列 ——
|
||||
progress_events: Annotated[List[dict], operator.add] # 累加,不覆盖
|
||||
```
|
||||
|
||||
### State 字段的特殊语义
|
||||
|
||||
- **`Annotated[..., operator.add]`** — LangGraph 的累加 reducer,节点 return 时 `progress_events: [evt]` 会**追加**到现有列表,而不是替换
|
||||
- **TypedDict** — 运行时是普通 dict,类型注解仅供 IDE 提示
|
||||
- **节点输出** — 节点 return 的 dict 只需要包含**要更新的字段**,未指定的字段保持不变
|
||||
|
||||
---
|
||||
|
||||
## 五、路由层(app/gateway/routers/ai_writing.py)
|
||||
|
||||
### 14 个接口一览
|
||||
|
||||
| HTTP | 路径 | 用途 | 鉴权 |
|
||||
|---|---|---|---|
|
||||
| POST | `/api/ai-writing/start` | 启动写作,返回 session_id | 用户 |
|
||||
| GET | `/api/ai-writing/{id}/stream` | SSE 进度流 | 用户 |
|
||||
| POST | `/api/ai-writing/{id}/resume` | 提交用户干预 | 用户 |
|
||||
| GET | `/api/ai-writing/{id}/state` | 查 graph 状态快照(调试用) | 用户 |
|
||||
| GET | `/api/ai-writing/sessions` | 列出当前用户的历史会话 | 用户 |
|
||||
| GET | `/api/ai-writing/sessions/{id}` | 获取单个历史会话 | 用户 |
|
||||
| PATCH | `/api/ai-writing/sessions/{id}` | 重命名会话 | 用户 |
|
||||
| DELETE | `/api/ai-writing/sessions/{id}` | 删除会话 | 用户 |
|
||||
| PUT | `/api/ai-writing/sessions/{id}/transcript` | 保存对话时间线(**fire-and-forget**) | 用户 |
|
||||
| GET | `/api/ai-writing/article-types` | 列出文章类型 | 用户 |
|
||||
| POST | `/api/ai-writing/article-types` | 新增文章类型 | 用户(建议改成 admin) |
|
||||
| PATCH | `/api/ai-writing/article-types/{id}` | 更新文章类型 | 用户 |
|
||||
| DELETE | `/api/ai-writing/article-types/{id}` | 删除文章类型 | 用户 |
|
||||
| POST | `/api/ai-writing/admin/cleanup` | **手动触发清理** | admin only |
|
||||
|
||||
### 模块级状态字典(关键设计 — 也是 worker 限制的根源)
|
||||
|
||||
```python
|
||||
# 每个 session_id 对应一份内存状态。多 worker 部署时每个 worker 一份,互不共享。
|
||||
_event_queues: dict[str, asyncio.Queue] # SSE 事件队列
|
||||
_session_pending_pause: dict[str, tuple] # 挂起的 await_user 事件(供重连重投)
|
||||
_session_model_names: dict[str, str] # 用户选的模型
|
||||
_session_interventions: dict[str, list] # 累计干预历史
|
||||
_session_repos: dict[str, repo] # DB 仓库引用
|
||||
_session_user_ids: dict[str, str] # 用户 id
|
||||
_session_article_types: dict[str, list] # 动态文章类型列表
|
||||
_session_error_flags: dict[str, bool]
|
||||
_inflight_graph_tasks: set[asyncio.Task] # in-flight task 注册表(lifespan drain 用)
|
||||
_recovery_lock: asyncio.Lock # 跨 worker 会话恢复的并发锁
|
||||
```
|
||||
|
||||
⚠️ **关键限制**:这些都是 per-process 内存。多 worker 部署下,worker B 看不到 worker A 的状态 → SSE 事件分发失败。详见 `frontend-web/docs/ai-writing-说明.md` 第五、六章。
|
||||
|
||||
### `_stream_graph_segment`(graph 推进函数)
|
||||
|
||||
`/start` 和 `/resume` 都调用这个函数(用 `_spawn_graph_task` 包装),负责:
|
||||
1. 调 `graph.astream(input_or_command, config=config, stream_mode="updates")`
|
||||
2. 把每个 chunk 解析成 SSE 事件 put 进 `_event_queues[session_id]`
|
||||
3. 遇到 `__interrupt__` → 推 `await_user` 事件 → 返回(不关闭 queue)
|
||||
4. 遇到 END → 推 `done` 事件 → 关闭 queue(put None 哨兵)
|
||||
5. 任何异常 → 推 `error` 事件 → 关闭 queue
|
||||
6. finally 段:更新 DB 最终状态、清理所有模块字典
|
||||
|
||||
### SSE 生成器 `_sse_generator`
|
||||
|
||||
`GET /stream` 的实现,从 `_event_queues[session_id]` 拿事件 → `_sse(event, data)` 序列化 → yield 给 fastapi `StreamingResponse`。
|
||||
|
||||
**关键逻辑**:
|
||||
- 队列不存在时**等待 5 秒**,期间触发 `_recover_session`(跨 worker / 进程重启恢复)
|
||||
- `_sse()` 三层兜底:`default=str` → 手写 notice 帧 → 永不抛 TypeError
|
||||
- 收到 None 哨兵 → 关闭流
|
||||
|
||||
### `_recover_session`(跨 worker 恢复)
|
||||
|
||||
当 worker 内存里没有 session_id 但 sqlite checkpointer 有快照时:
|
||||
1. 从 checkpointer 读出 state snapshot
|
||||
2. 重建 `_event_queues[sid]` 队列
|
||||
3. 从 snapshot 提取当前 interrupt 负载,put 进队列
|
||||
4. 同时尽量恢复 `_session_repos`、`_session_user_ids` 等
|
||||
|
||||
返回三种状态:
|
||||
- `"interrupted"` — 图停在干预点,已重建队列
|
||||
- `"finished"` — 图已完成或中断在节点中间,推 done/error 事件后关闭
|
||||
- `"unrecoverable"` — sqlite 也没有这个 thread_id → 真的丢了
|
||||
|
||||
### `_spawn_graph_task` / `wait_for_inflight_graph_tasks`(优雅关闭)
|
||||
|
||||
- `_spawn_graph_task(coro)` — 启 task 并自动登记到 `_inflight_graph_tasks`,完成时反登记
|
||||
- `wait_for_inflight_graph_tasks(timeout)` — lifespan shutdown 等所有 task 跑到下一个 interrupt / END(默认 30s),超时则 cancel 强制结束
|
||||
|
||||
---
|
||||
|
||||
## 六、Graph 与持久化(graph.py)
|
||||
|
||||
### Lazy 单例 + 自动降级
|
||||
|
||||
```python
|
||||
AI_WRITING_GRAPH = None # 模块级槽位(测试可 monkeypatch)
|
||||
_async_checkpointer_singleton = None # checkpointer 单例
|
||||
|
||||
async def get_ai_writing_graph():
|
||||
"""懒加载 graph。第一次调用时按 config.yaml 决定 checkpointer。"""
|
||||
global AI_WRITING_GRAPH
|
||||
if AI_WRITING_GRAPH is not None:
|
||||
return AI_WRITING_GRAPH
|
||||
checkpointer = await _get_async_checkpointer()
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
|
||||
return AI_WRITING_GRAPH
|
||||
```
|
||||
|
||||
### Checkpointer 优先级(_get_async_checkpointer)
|
||||
|
||||
```
|
||||
1. config.yaml 的 checkpointer 段
|
||||
├── type: sqlite → AsyncSqliteSaver(aiosqlite.connect(...))
|
||||
├── type: postgres → AsyncPostgresSaver(AsyncConnectionPool(...))
|
||||
└── type: memory → InMemorySaver
|
||||
|
||||
2. 任何初始化失败 → InMemorySaver(降级,打 warning)
|
||||
3. 未配置 checkpointer 段 → InMemorySaver(降级,打 warning)
|
||||
4. 未知 type(如 "mysql") → InMemorySaver(降级,打 warning)
|
||||
```
|
||||
|
||||
**核心契约**:`_get_async_checkpointer()` **绝不抛异常**,永远返回一个可用的 saver。这是「写作流程能完整继续」契约的关键。
|
||||
|
||||
### Shutdown
|
||||
|
||||
`shutdown_ai_writing_graph()` 在 lifespan shutdown 时调用:
|
||||
- 关闭 `aiosqlite.Connection` / `AsyncConnectionPool`
|
||||
- 清空 `AI_WRITING_GRAPH` / `_async_checkpointer_singleton`
|
||||
|
||||
调用顺序(详见 `app/gateway/app.py` lifespan):
|
||||
1. 停清理任务
|
||||
2. `wait_for_inflight_graph_tasks(30s)`
|
||||
3. `shutdown_ai_writing_graph()`
|
||||
|
||||
---
|
||||
|
||||
## 七、节点详解(nodes/)
|
||||
|
||||
### `nodes/_utils.py`(公共工具)
|
||||
|
||||
| 函数 / 类 | 用途 |
|
||||
|---|---|
|
||||
| `get_model(config, model_name)` | 从 config.configurable 拿 model_name 或回退默认 |
|
||||
| `parse_json(text, fallback)` | 鲁棒 JSON 解析(支持代码块、不完整 JSON、转义错误) |
|
||||
| `_extract_balanced(text)` | 找出第一对平衡的 `{...}` |
|
||||
| `_repair_json(text)` | 修复常见的 JSON 错误(缺尾引号等) |
|
||||
| `ThinkTagParser` | 流式解析 `<think>...</think>` 标签(思考 / 正文分流) |
|
||||
| `llm_json(system, user, fallback, config)` | 调 LLM 并解析 JSON 输出 |
|
||||
| `count_words(text)` | 中文 + 英文混合计字数 |
|
||||
|
||||
### `nodes/intent_parser.py` — 意图解析
|
||||
|
||||
- 调 LLM 把用户意图(`user_intent` 字符串)解析成结构化字段:`article_type`、`target_audience`、`word_count_target`
|
||||
- 用户已经在 `AIWritingRequest` 里指定的字段会**覆盖** LLM 推断
|
||||
- 输出:更新 state 的对应字段,推 `step_done` 事件
|
||||
|
||||
### `nodes/researcher.py` — 素材检索
|
||||
|
||||
- 根据 `user_intent` 用 LLM 生成 N 个关键词(N = `keyword_count`,默认 6)
|
||||
- 用 `search/*` 的 provider 并行检索
|
||||
- 流式推 `research_progress` 事件("正在搜索关键词 X 中...")
|
||||
- 输出:`material_package = {keywords, materials, summary}`,推 `materials_ready`
|
||||
|
||||
### `nodes/writer_outline.py` — 大纲规划
|
||||
|
||||
- 输入:`user_intent` + `material_package`
|
||||
- 调 LLM 流式生成大纲 JSON,期间推 `outline_chunk`(正文 + thinking 分流)
|
||||
- 解析最终 JSON → `current_outline`,推 `outline_ready`
|
||||
|
||||
### `nodes/writer_draft.py` — 草稿生成(最复杂)
|
||||
|
||||
- 按大纲**逐章节**调 LLM 生成正文,每章节独立流式推 `draft_chunk`
|
||||
- 检测 LLM 是否「拒答」(`_looks_like_refusal`),如果是则换提示重试
|
||||
- **严格模式下**:如果章节关联的素材不足,把该章节加入 `blocked_sections`,触发 `pause_section_help`
|
||||
- 完成所有章节后拼成 `current_draft.full_markdown`,推 `draft_ready`
|
||||
|
||||
### `nodes/editor.py` — 编辑审核(3 维度并行)
|
||||
|
||||
- 3 个维度(**fact / logic / language**)并行调 LLM
|
||||
- 每个维度独立流式推 `review_chunk`,完成后推 `review_item_done`
|
||||
- 三个都完成后聚合成 `review_result`,推 `review_ready`
|
||||
- 评分加权(fact 40% + logic 30% + language 30%),超过 `pass_threshold`(默认 80)verdict = `pass`
|
||||
|
||||
---
|
||||
|
||||
## 八、持久化(persistence/ai_writing_sessions/)
|
||||
|
||||
### `model.py` — ORM
|
||||
|
||||
```python
|
||||
class AIWritingSessionRow(Base):
|
||||
__tablename__ = "ai_writing_sessions"
|
||||
id: Mapped[str] # = session_id(uuid)
|
||||
user_id: Mapped[str | None] # NULL = no-auth 模式
|
||||
title: Mapped[str] # 从 user_intent 截取前 50 字
|
||||
user_intent: Mapped[str | None]
|
||||
status: Mapped[str] # in_progress / draft_ready / review_ready / done / error
|
||||
draft_title: Mapped[str | None]
|
||||
draft_markdown: Mapped[str | None]
|
||||
completed_interventions: Mapped[str | None] # JSON 字符串
|
||||
review_result: Mapped[str | None] # JSON 字符串
|
||||
transcript: Mapped[str | None] # PortableLongText(MySQL→LONGTEXT,4GB)
|
||||
created_at / updated_at: BeijingDateTime
|
||||
```
|
||||
|
||||
### `sql.py` — Repository
|
||||
|
||||
| 方法 | 用途 |
|
||||
|---|---|
|
||||
| `create(id, title, user_intent, user_id)` | 启动写作时建一行 |
|
||||
| `get(session_id, user_id)` | 取单个(带 user_id 权限校验) |
|
||||
| `list(user_id, limit)` | 列出当前用户的会话 |
|
||||
| `update(session_id, user_id, ...)` | 更新指定字段(其他保持不变) |
|
||||
| `delete(session_id, user_id)` | 删除(带权限校验) |
|
||||
| `list_older_than(cutoff, only_finished, limit)` | 清理用:列出过期 session_id |
|
||||
| `delete_by_ids(session_ids)` | 清理用:批量删除 |
|
||||
|
||||
### Alembic 迁移
|
||||
|
||||
- `20260517_03_ai_writing_transcript.py` — 加 transcript 列(TEXT)
|
||||
- `20260519_02_ai_writing_transcript_longtext.py` — MySQL 上把 transcript 改成 LONGTEXT(修 64KB 限制导致的 500)
|
||||
|
||||
---
|
||||
|
||||
## 九、定期清理(ai_writing_cleanup.py)
|
||||
|
||||
### 设计目标
|
||||
- 删超过 `retention_days`(默认 7)的旧会话
|
||||
- 联动清理:MySQL 业务行 + SQLite checkpointer 里的 thread 状态
|
||||
- **绝对不删正在写的会话**(活跃保护)
|
||||
|
||||
### 关键函数
|
||||
|
||||
```python
|
||||
async def run_cleanup_once(*, retention_days, only_finished=False, batch_size=500) -> dict
|
||||
```
|
||||
|
||||
返回 `{deleted_db, deleted_checkpoints, skipped_active}`。
|
||||
|
||||
执行流程:
|
||||
1. 调 `repo.list_older_than(cutoff, only_finished, batch_size)` 拿过期 id
|
||||
2. 从 `_event_queues.keys()` 拿活跃 session_id 集合 → **过滤掉所有仍在跑的**
|
||||
3. 对每个待删 id:调 `checkpointer.adelete_thread(id)` 删 checkpoint
|
||||
4. 调 `repo.delete_by_ids(ids)` 批量删 MySQL
|
||||
5. 循环直到本批 < batch_size
|
||||
|
||||
### Cron 调度
|
||||
|
||||
`cleanup_scheduler_loop()` 常驻后台任务:
|
||||
- 用 `croniter` 解析 `ai_writing.cleanup.cron`(默认 `"0 3 * * *"`,每天凌晨 3 点)
|
||||
- `asyncio.sleep` 到下次触发时间
|
||||
- 多 worker 用文件锁(`runtime_home() / ".ai_writing_cleanup.lock"`)做 leader election
|
||||
|
||||
### Admin 手动触发
|
||||
|
||||
`POST /api/ai-writing/admin/cleanup`:
|
||||
- 鉴权:`system_role == "admin"`
|
||||
- 可选覆盖 `retention_days` / `only_finished` / `batch_size`
|
||||
- 即使传 `retention_days: 0` 也不会秒杀活跃会话(活跃保护硬保证)
|
||||
|
||||
---
|
||||
|
||||
## 十、配置(config.yaml)
|
||||
|
||||
```yaml
|
||||
# Checkpointer:graph 状态持久化
|
||||
checkpointer:
|
||||
type: sqlite # sqlite | postgres | memory
|
||||
connection_string: .deer-flow/data/checkpoints.db
|
||||
|
||||
# 清理调度
|
||||
ai_writing:
|
||||
cleanup:
|
||||
enabled: true
|
||||
retention_days: 7
|
||||
cron: "0 3 * * *" # 每天凌晨 3 点
|
||||
delete_only_finished: false
|
||||
batch_size: 500
|
||||
|
||||
# (可选)素材检索 provider 配置
|
||||
search:
|
||||
provider: deerflow.agents.ai_writing.search.internet_search:WebSearchProvider
|
||||
...
|
||||
```
|
||||
|
||||
`_ai_writing_config()` 函数从 `get_app_config().ai_writing` 里读 dict,节点通过 `config.configurable.ai_writing_config` 拿到。
|
||||
|
||||
---
|
||||
|
||||
## 十一、SSE 事件协议(后端 → 前端契约)
|
||||
|
||||
后端通过 `_sse(event_type, data)` 推 `event: <type>\ndata: <json>\n\n` 格式。
|
||||
|
||||
### 推出事件的位置
|
||||
|
||||
| 事件 | 推出位置 |
|
||||
|---|---|
|
||||
| `connected` | `_sse_generator` 入口 |
|
||||
| `step_start` / `step_done` | 节点内部主动 emit |
|
||||
| `research_progress` | `researcher_node` 内的 `_emit` |
|
||||
| `materials_ready` | `_stream_graph_segment` 检测到 `material_package` 字段 |
|
||||
| `outline_chunk` | `writer_outline_node` 流式推(通过 `sse_queue`) |
|
||||
| `outline_ready` | `_stream_graph_segment` 检测到 `current_outline` |
|
||||
| `draft_chunk` | `writer_draft_node` 按章节流式推 |
|
||||
| `draft_ready` | `_stream_graph_segment` 检测到 `current_draft` |
|
||||
| `review_chunk` / `review_item_done` | `editor_node` 3 维度并行流式 |
|
||||
| `review_ready` | `_stream_graph_segment` 检测到 `review_result` |
|
||||
| `await_user` | `_stream_graph_segment` 捕获 `__interrupt__` |
|
||||
| `resumed` | pause 节点 return 时塞进 `progress_events` |
|
||||
| `notice` | 节点内手动推(非致命告警) |
|
||||
| `done` | `_stream_graph_segment` 正常结束 |
|
||||
| `error` | `_stream_graph_segment` except 分支 |
|
||||
|
||||
### 节点如何流式推 chunk
|
||||
|
||||
通过 `config.configurable.sse_queue`:
|
||||
|
||||
```python
|
||||
async def writer_outline_node(state, config):
|
||||
sse_queue = config.get("configurable", {}).get("sse_queue")
|
||||
sse_session_id = config.get("configurable", {}).get("sse_session_id")
|
||||
|
||||
async for chunk in llm.astream(prompt):
|
||||
if sse_queue:
|
||||
await sse_queue.put(("outline_chunk", {
|
||||
"session_id": sse_session_id,
|
||||
"text": chunk.content,
|
||||
"is_thinking": False,
|
||||
}))
|
||||
|
||||
return {"current_outline": parsed, ...}
|
||||
```
|
||||
|
||||
`sse_queue` 是路由层在 `_make_config` 里注入的 `_event_queues[session_id]`。
|
||||
|
||||
---
|
||||
|
||||
## 十二、扩展指南:常见开发任务
|
||||
|
||||
### 任务 1:新增一个 Graph 节点
|
||||
|
||||
**例**:在 `editor` 之后加一个「术语统一」节点。
|
||||
|
||||
1. **写节点**:在 `nodes/` 新建 `terminology.py`:
|
||||
```python
|
||||
async def terminology_node(state, config):
|
||||
# ... 读 current_draft 改术语
|
||||
return {
|
||||
"current_draft": new_draft,
|
||||
"progress_events": [{"type": "step_done", "agent_name": "术语统一", "message": "..."}],
|
||||
}
|
||||
```
|
||||
|
||||
2. **注册到 graph**:在 `graph.py` 的 `build_ai_writing_graph` 里:
|
||||
```python
|
||||
from .nodes.terminology import terminology_node
|
||||
g.add_node("terminology", terminology_node)
|
||||
# 改 editor 后的边
|
||||
g.add_edge("editor", "terminology")
|
||||
g.add_edge("terminology", "pause_review")
|
||||
```
|
||||
|
||||
3. **State 加字段**(如果需要):`state.py` 加新字段到 `AIWritingState`
|
||||
|
||||
4. **前端事件支持**(如果要新事件类型):见前端文档「扩展指南」
|
||||
|
||||
5. **测试**:在 `tests/` 加单元测试,至少测节点能正常 return 和 graph 拓扑能编译
|
||||
|
||||
### 任务 2:新增一个用户干预点
|
||||
|
||||
1. **写 pause 节点**:在 `graph.py` 顶部加:
|
||||
```python
|
||||
async def pause_xxx(state):
|
||||
user_input = interrupt({
|
||||
"pause_point": "xxx_confirm",
|
||||
"message": "...",
|
||||
"context_field": state["..."],
|
||||
})
|
||||
action = user_input.get("action", "confirm")
|
||||
if action == "confirm":
|
||||
return {"status": "next_status", "progress_events": [{"type": "resumed", ...}]}
|
||||
# ... 其他分支
|
||||
```
|
||||
|
||||
2. **路由层 ResumeRequest 加字段**(如果干预需要新字段):
|
||||
```python
|
||||
class ResumeRequest(BaseModel):
|
||||
# ...
|
||||
new_field: Optional[str] = None
|
||||
```
|
||||
|
||||
3. **`resume_writing` 路由把字段塞进 Command(resume=...)**:
|
||||
```python
|
||||
intervention = {
|
||||
# ...
|
||||
"newField": body.new_field,
|
||||
}
|
||||
```
|
||||
|
||||
4. **graph 拓扑加 pause 节点的边**:在 build_ai_writing_graph 里
|
||||
|
||||
5. **前端配合**:见前端文档
|
||||
|
||||
### 任务 3:新增一种素材检索 provider
|
||||
|
||||
1. 在 `agents/ai_writing/search/` 加新文件,实现 `SearchProvider` 接口(参考 `internet_search.py`):
|
||||
```python
|
||||
class MyProvider(SearchProvider):
|
||||
async def search(self, keyword: str, limit: int) -> list[SearchResult]:
|
||||
...
|
||||
```
|
||||
|
||||
2. 配置 `config.yaml`:
|
||||
```yaml
|
||||
ai_writing:
|
||||
search:
|
||||
provider: my_pkg.search:MyProvider
|
||||
# provider 特定的配置...
|
||||
```
|
||||
|
||||
3. `researcher_node` 通过 `config.configurable.ai_writing_config.search.provider` 用 `resolve_class` 动态加载
|
||||
|
||||
### 任务 4:修改 LLM Prompt
|
||||
|
||||
所有 prompt 集中在 `agents/ai_writing/prompts/`:
|
||||
- `researcher_prompts.py` — 检索专家
|
||||
- `writer_prompts.py` — 大纲 + 草稿
|
||||
- `editor_prompts.py` — 审核
|
||||
|
||||
改 prompt 后**强烈建议**:
|
||||
1. 在 `tests/` 加针对该 prompt 输出的回归测试(用 LLM mock 或固定输入)
|
||||
2. 跑现有测试确认拓扑兼容
|
||||
|
||||
### 任务 5:扩展持久化字段
|
||||
|
||||
例:给 `ai_writing_sessions` 加一个 `total_tokens` 列。
|
||||
|
||||
1. `model.py` 加列:
|
||||
```python
|
||||
total_tokens: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
```
|
||||
|
||||
2. **必须**新建 alembic 迁移:
|
||||
```bash
|
||||
PYTHONPATH=. uv run alembic revision -m "add total_tokens column"
|
||||
```
|
||||
然后填 upgrade/downgrade 逻辑(参考 `20260519_02_ai_writing_transcript_longtext.py`)
|
||||
|
||||
3. `sql.py` 的 `update()` / `to_dict()` 加字段处理
|
||||
|
||||
4. `routers/ai_writing.py` 的 `SessionResponse` 加字段
|
||||
|
||||
5. 测试:在 `tests/test_ai_writing_session_repo.py` 加新字段的 round-trip 测试
|
||||
|
||||
---
|
||||
|
||||
## 十三、容错策略(关键设计原则)
|
||||
|
||||
### 「写作流程能完整继续」契约
|
||||
|
||||
后端承诺:**除非 Pydantic 校验失败(4xx),AI 写作的 4 个核心接口任何后端组件失败都不阻断当次写作的完整执行**。
|
||||
|
||||
### 落实位置
|
||||
|
||||
| 失败点 | 兜底位置 | 行为 |
|
||||
|---|---|---|
|
||||
| sqlite checkpointer 初始化失败 | `_get_async_checkpointer` | 自动降级 InMemorySaver |
|
||||
| SSE JSON 序列化失败 | `_sse()` | 三层兜底(default=str → notice 帧) |
|
||||
| DB `create()` session 行失败 | `start_writing` | try/except 吞掉,session_id 仍返回 |
|
||||
| DB `update(draft)` 失败 | `_stream_graph_segment` | try/except 吞掉,graph 继续推 |
|
||||
| DB `update(review)` 失败 | 同上 | 同上 |
|
||||
| DB `update(transcript)` 失败 | `save_transcript` 路由 | 返回 `success=false`,不抛 5xx |
|
||||
| 文章类型加载失败 | `start_writing` | 退回 `[]` |
|
||||
| Graph 编译失败 | `start_writing` | 抛 503(极端情况,节点 import 错) |
|
||||
| Graph 运行时节点抛错 | `_stream_graph_segment` | except 推 error 事件,session 结束 |
|
||||
| Cleanup 误删活跃会话 | `run_cleanup_once` | 活跃保护硬过滤 |
|
||||
|
||||
### 容器优雅关闭
|
||||
|
||||
lifespan shutdown 顺序:
|
||||
1. 停 cleanup scheduler
|
||||
2. `wait_for_inflight_graph_tasks(30s)` — 等所有 in-flight 写作跑到下一个 checkpoint
|
||||
3. 超时未结束的 task `cancel()` + 2s ack 等待
|
||||
4. `shutdown_ai_writing_graph()` — 关闭 aiosqlite Connection / pg pool
|
||||
|
||||
---
|
||||
|
||||
## 十四、测试覆盖
|
||||
|
||||
| 测试文件 | 数量 | 覆盖范围 |
|
||||
|---|---|---|
|
||||
| `tests/test_ai_writing_session_repo.py` | 7 | Repo CRUD + 清理用的 list_older_than / delete_by_ids |
|
||||
| `tests/test_ai_writing_session_recovery.py` | 8 | `_recover_session` / `_extract_interrupt` / `resume_writing` 跨 worker 恢复 |
|
||||
| `tests/test_ai_writing_transcript_endpoint.py` | 5 | transcript 保存兜底 |
|
||||
| `tests/test_ai_writing_cleanup.py` | 26 | 清理 + drain + checkpointer 降级 + SSE 序列化 + admin 接口 + 多 worker 检测 |
|
||||
| `tests/test_portable_long_text.py` | 5 | LONGTEXT 列类型 + 迁移链 |
|
||||
|
||||
跑全部:
|
||||
```bash
|
||||
PYTHONPATH=. uv run pytest tests/test_ai_writing*.py tests/test_portable_long_text.py -v
|
||||
```
|
||||
|
||||
期望 **51 个测试全过**。
|
||||
|
||||
---
|
||||
|
||||
## 十五、关键调试 / 排查命令
|
||||
|
||||
### 看 graph 当前状态
|
||||
|
||||
```python
|
||||
# 进 python shell
|
||||
from deerflow.agents.ai_writing.graph import get_ai_writing_graph
|
||||
graph = await get_ai_writing_graph()
|
||||
snap = await graph.aget_state({"configurable": {"thread_id": "<session_id>"}})
|
||||
print(snap.values, snap.next, snap.tasks)
|
||||
```
|
||||
|
||||
### 直接读 sqlite checkpointer
|
||||
|
||||
```bash
|
||||
sqlite3 /app/.deer-flow/data/checkpoints.db
|
||||
.tables # 看表(checkpoints / checkpoint_writes 等)
|
||||
SELECT thread_id, checkpoint_ns, checkpoint_id FROM checkpoints
|
||||
WHERE thread_id = '<session_id>' LIMIT 5;
|
||||
```
|
||||
|
||||
### 查 MySQL 业务行
|
||||
|
||||
```sql
|
||||
SELECT id, status, draft_title, updated_at,
|
||||
JSON_LENGTH(transcript) AS transcript_size
|
||||
FROM ai_writing_sessions
|
||||
WHERE id = '<session_id>';
|
||||
```
|
||||
|
||||
### 查内存中的活跃会话(调试时手动加)
|
||||
|
||||
```python
|
||||
# 临时在路由加一个 admin 接口
|
||||
@router.get("/admin/inflight")
|
||||
async def admin_inflight():
|
||||
return {
|
||||
"event_queues": list(_event_queues.keys()),
|
||||
"inflight_tasks": len(_inflight_graph_tasks),
|
||||
"pending_pause": list(_session_pending_pause.keys()),
|
||||
}
|
||||
```
|
||||
|
||||
### 日志关键字
|
||||
|
||||
```bash
|
||||
docker logs <gateway> 2>&1 | grep -E 'ai_writing|AsyncSqliteSaver|checkpointer|recovered'
|
||||
```
|
||||
|
||||
期望看到(启动时):
|
||||
- `ai_writing: 使用 AsyncSqliteSaver (xxx.db)` — 持久化生效
|
||||
- `ai_writing_cleanup: 已启动 cron=...` — 清理调度起来
|
||||
|
||||
警告:
|
||||
- `ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver` — sqlite 故障
|
||||
- `⚠ 检测到 WEB_CONCURRENCY=N` — 多 worker 警告(详见 `ai-writing-说明.md`)
|
||||
|
||||
---
|
||||
|
||||
## 十六、性能与限制
|
||||
|
||||
### 单 session 资源占用
|
||||
- 内存:~50KB(state + progress_events 累计)
|
||||
- sqlite checkpoint:每次 state update 写一行(一篇完整文章约 20-50 行)
|
||||
- LLM tokens:约 30K-100K(取决于素材量 + 字数 + 修订次数)
|
||||
|
||||
### 并发上限
|
||||
- 单 worker 内:理论上百级 session 并发(IO bound),实际 LLM API rate limit 才是瓶颈
|
||||
- 多 worker:**当前 in-memory dict 隔离,详见 ai-writing-说明.md 第六章**
|
||||
|
||||
### 长会话风险
|
||||
- progress_events 累加无界 → 长会话 state 可能膨胀到几 MB
|
||||
- transcript JSON 同理 → MySQL 行可能上 MB(但 LONGTEXT 列上限 4GB,影响读取性能)
|
||||
|
||||
### 优化方向
|
||||
- progress_events 节流(聚合 chunk 事件)
|
||||
- transcript 增量保存(diff 而非全量覆盖)
|
||||
- 节点级 LLM 调用加 retry(tenacity)
|
||||
|
||||
---
|
||||
|
||||
## 十七、与主 Agent 的关系
|
||||
|
||||
| 维度 | 主 Agent | AI 写作 |
|
||||
|---|---|---|
|
||||
| Graph 入口 | `make_lead_agent` | `AI_WRITING_GRAPH` |
|
||||
| Middleware | 18 层中间件 | 无(直接节点链) |
|
||||
| State | `ThreadState` | `AIWritingState` |
|
||||
| Checkpointer | `make_checkpointer()`(共享) | `_get_async_checkpointer()`(同一文件,不同 thread_id 命名空间) |
|
||||
| 路由前缀 | `/api/langgraph/*` `/api/threads/*` | `/api/ai-writing/*` |
|
||||
| 流式协议 | LangGraph stream_mode(messages-tuple/values) | 自定义 SSE 事件 |
|
||||
| 用户干预 | 通过 `ask_clarification` 工具 | 通过 LangGraph `interrupt()` |
|
||||
| 持久化业务表 | `threads_meta`、`runs` | `ai_writing_sessions` |
|
||||
|
||||
二者**完全独立**,仅在「共享同一个 sqlite checkpointer 文件」这一点上有间接关联(也可以拆开,性能层面没区别)。
|
||||
|
||||
---
|
||||
|
||||
## 十八、未来 RFC 方向
|
||||
|
||||
| 主题 | 描述 |
|
||||
|---|---|
|
||||
| 多 worker 部署支持 | 把 `_event_queues` 等搬 Redis Pub/Sub,参考 `ai-writing-说明.md` 方案 B |
|
||||
| Graph 拓扑可配置 | 现在节点链路写死,未来按 article_type 走不同拓扑(如「研报」vs「散文」) |
|
||||
| 节点级重试 | LLM 抽风时自动重试 N 次,参考 `tenacity` |
|
||||
| 素材跨 session 复用 | 同一用户多次写作可能用同一批素材 → 缓存层 |
|
||||
| 多模态素材 | 支持图片素材输入和图文混合输出 |
|
||||
| 中文标点 / 排版规则注入 | 在节点 prompt 里强化中文出版规范 |
|
||||
|
||||
---
|
||||
|
||||
## 附录 A:相关文件速查
|
||||
|
||||
| 文件 | 行数 | 用途 |
|
||||
|---|---|---|
|
||||
| `routers/ai_writing.py` | 1006 | 路由 + 状态字典 + SSE + drain |
|
||||
| `agents/ai_writing/graph.py` | 538 | StateGraph + checkpointer 工厂 |
|
||||
| `agents/ai_writing/state.py` | 121 | AIWritingState 定义 |
|
||||
| `agents/ai_writing/nodes/_utils.py` | 242 | LLM 工具 |
|
||||
| `agents/ai_writing/nodes/intent_parser.py` | 131 | 节点:意图解析 |
|
||||
| `agents/ai_writing/nodes/researcher.py` | 271 | 节点:素材检索 |
|
||||
| `agents/ai_writing/nodes/writer_outline.py` | 147 | 节点:大纲规划 |
|
||||
| `agents/ai_writing/nodes/writer_draft.py` | 318 | 节点:草稿生成 |
|
||||
| `agents/ai_writing/nodes/editor.py` | 180 | 节点:编辑审核 |
|
||||
| `app/gateway/ai_writing_cleanup.py` | ~280 | 清理 + 调度 |
|
||||
| `persistence/ai_writing_sessions/sql.py` | ~200 | Repository |
|
||||
| `persistence/ai_writing_sessions/model.py` | ~35 | ORM |
|
||||
|
||||
---
|
||||
|
||||
## 附录 B:常见踩坑
|
||||
|
||||
| 现象 | 根因 | 解决 |
|
||||
|---|---|---|
|
||||
| `/start` 报 500 | graph 编译失败(多半是 import 错) | 看启动日志 stack trace |
|
||||
| `/resume` 报 404 | session_id 不存在或 checkpointer 没快照 | 看 `ai_writing recover` 日志 |
|
||||
| transcript 500 | DB 列容量 / 网络抖 | ✅ 已修,前端 fire-and-forget |
|
||||
| SSE 卡住不动 | 多 worker SSE 隔离 / 反代超时 | 见 `ai-writing-说明.md` |
|
||||
| `draft_chunk` 丢失内容 | `ThinkTagParser` 在 `<think>` 标签边界处理 | 看 `_utils.py:ThinkTagParser.feed` 单测 |
|
||||
| graph 卡在某节点不退出 | LLM API 卡住 / 无限重试 | 看节点日志,加 timeout |
|
||||
| cleanup 没跑 | cron 表达式配错 / 文件锁被其他 worker 持有 | 看 `ai_writing_cleanup` 日志 |
|
||||
| 草稿在前端看不到但 MySQL 有 | 多 worker 导致 SSE 事件丢失 | 同上 |
|
||||
| 配 mysql 做 checkpointer 报错 | LangGraph 不支持 MySQL checkpointer | 用 sqlite 或 postgres |
|
||||
|
||||
---
|
||||
|
||||
如需补充内容,请直接编辑本文件,并在 git commit message 里注明改动原因。
|
||||
@@ -0,0 +1,278 @@
|
||||
# 公开定时任务 API
|
||||
|
||||
本文档描述用于对接外部系统的**公开只读**定时任务接口,以及配套的公开网页入口。所有 `/api/public/*` 接口都**无需登录**,可以直接调用。
|
||||
|
||||
> ⚠️ 任何接入这套接口的系统需要确认:暴露 task / run 列表与正文是符合贵方业务安全要求的。如果不希望全员可见,请走带鉴权的 `/api/scheduled-tasks/*`。
|
||||
|
||||
---
|
||||
|
||||
## 1. 基础信息
|
||||
|
||||
| 项 | 值 |
|
||||
|---|---|
|
||||
| 网关基址(开发) | `http://localhost:8001` |
|
||||
| 网关基址(生产) | 部署到 Nginx 后通常是 `https://<你的域名>` |
|
||||
| 公开 API 前缀 | `/api/public/scheduled-tasks` |
|
||||
| 鉴权 | **无需**鉴权(白名单在 `auth_middleware._PUBLIC_PATH_PREFIXES`) |
|
||||
| 编码 | 请求 / 响应均为 UTF-8,响应类型 `application/json`(下载除外) |
|
||||
| 错误响应 | 标准 FastAPI 错误体 `{"detail": "..."}`;HTTP 状态码 `404 / 400 / 500` 等 |
|
||||
|
||||
公开网页页面:
|
||||
|
||||
```
|
||||
#/public/scheduled-tasks/<task_id>
|
||||
```
|
||||
|
||||
例如 `https://<你的域名>/#/public/scheduled-tasks/8f1d8b9c-...`。
|
||||
左侧列出该任务所有执行记录,默认打开第一条;右侧默认渲染 Markdown 正文,点击 Tab 可以查看“结果文件”或“元数据”,点击左侧任意记录右侧内容会跟随切换。
|
||||
|
||||
---
|
||||
|
||||
## 2. 接口列表
|
||||
|
||||
### 2.1 列出全部定时任务
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks
|
||||
```
|
||||
|
||||
**Query 参数**
|
||||
|
||||
| 名称 | 类型 | 必填 | 默认 | 说明 |
|
||||
|---|---|---|---|---|
|
||||
| `search` | string | 否 | — | 任务名模糊匹配(不区分大小写) |
|
||||
| `limit` | int | 否 | 200 | 单页数量,1–500 |
|
||||
| `offset` | int | 否 | 0 | 偏移量,配合 `limit` 分页 |
|
||||
|
||||
**响应**
|
||||
|
||||
```json
|
||||
{
|
||||
"total": 12,
|
||||
"limit": 200,
|
||||
"offset": 0,
|
||||
"tasks": [
|
||||
{
|
||||
"task_id": "8f1d8b9c-...",
|
||||
"name": "每日舆情简报",
|
||||
"schedule_text": "每天 09:00",
|
||||
"cron_expr": "0 9 * * *",
|
||||
"timezone": "Asia/Shanghai",
|
||||
"enabled": true,
|
||||
"next_run_at": "2026-05-21T01:00:00+00:00",
|
||||
"last_run_at": "2026-05-20T01:00:12+00:00",
|
||||
"last_status": "succeeded",
|
||||
"published": true
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> 已脱敏:`user_id`、`creator_email`、`prompt`、`scheduler_thread_id`、`execution_context` 等私有字段不会出现在公开响应里。
|
||||
|
||||
### 2.2 获取单个任务元数据
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/{task_id}
|
||||
```
|
||||
|
||||
**响应**:与上面 `tasks[]` 的元素结构一致;任务不存在返回 `404`。
|
||||
|
||||
### 2.3 列出某任务的所有执行记录
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/{task_id}/runs
|
||||
```
|
||||
|
||||
**Query 参数**
|
||||
|
||||
| 名称 | 类型 | 必填 | 默认 | 说明 |
|
||||
|---|---|---|---|---|
|
||||
| `limit` | int | 否 | 50 | 最近执行数量,1–200 |
|
||||
|
||||
**响应**
|
||||
|
||||
```json
|
||||
{
|
||||
"task": { "task_id": "...", "name": "...", "schedule_text": "...", "timezone": "...", "enabled": true },
|
||||
"total": 7,
|
||||
"runs": [
|
||||
{
|
||||
"id": "run-uuid-...",
|
||||
"task_id": "8f1d8b9c-...",
|
||||
"scheduled_for": "2026-05-20T01:00:00+00:00",
|
||||
"started_at": "2026-05-20T01:00:01+00:00",
|
||||
"finished_at": "2026-05-20T01:00:12+00:00",
|
||||
"status": "succeeded",
|
||||
"error": null,
|
||||
"retry_count": 0,
|
||||
"result": {
|
||||
"title": "2026-05-20 舆情简报",
|
||||
"content": "# 简报\n\n...",
|
||||
"format": "markdown",
|
||||
"files": [
|
||||
{ "index": 0, "name": "report.md", "mime_type": "text/markdown", "kind": "markdown", "size": 4321 }
|
||||
],
|
||||
"created_at": "2026-05-20T01:00:12+00:00"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> 已脱敏:返回的 `files[]` 不包含服务器绝对路径;客户端通过下标 `index` 调用文件接口。
|
||||
|
||||
### 2.4 获取单条执行记录
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/runs/{run_id}
|
||||
```
|
||||
|
||||
返回单条 run 数据(结构同 2.3 中的 `runs[]` 元素)。
|
||||
|
||||
### 2.5 获取单个结果文件的内容(可预览文本直接内联)
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/runs/{run_id}/files/{file_index}
|
||||
```
|
||||
|
||||
`file_index` 来自 `run.result.files[].index`。
|
||||
|
||||
**响应**
|
||||
|
||||
```json
|
||||
{
|
||||
"index": 0,
|
||||
"name": "report.md",
|
||||
"mime_type": "text/markdown",
|
||||
"kind": "markdown",
|
||||
"size": 4321,
|
||||
"previewable": true,
|
||||
"content": "# 简报\n\n..."
|
||||
}
|
||||
```
|
||||
|
||||
二进制文件 `previewable=false`、`content=null`,请走 2.6 下载。
|
||||
|
||||
### 2.6 下载单个结果文件
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/runs/{run_id}/files/{file_index}/download
|
||||
```
|
||||
|
||||
返回原始文件二进制流,附带 `Content-Disposition: attachment` 头部。
|
||||
|
||||
### 2.7 打包下载整个执行结果
|
||||
|
||||
```
|
||||
GET /api/public/scheduled-tasks/runs/{run_id}/download
|
||||
```
|
||||
|
||||
- 如果该 run **没有附加文件**:直接返回 `result.md`(UTF-8 BOM)。
|
||||
- 如果有 ≥ 1 个附加文件:返回 ZIP,里面包含:
|
||||
- `result.md` — 标题 + 正文 Markdown
|
||||
- 每个生成文件原样
|
||||
|
||||
---
|
||||
|
||||
## 3. 调用示例
|
||||
|
||||
### curl
|
||||
|
||||
```bash
|
||||
# 列出全部任务
|
||||
curl -s "https://<domain>/api/public/scheduled-tasks?limit=50"
|
||||
|
||||
# 查看任务的执行记录
|
||||
curl -s "https://<domain>/api/public/scheduled-tasks/<task_id>/runs"
|
||||
|
||||
# 下载某次执行
|
||||
curl -L -o report.zip \
|
||||
"https://<domain>/api/public/scheduled-tasks/runs/<run_id>/download"
|
||||
```
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
base = "https://<domain>/api/public/scheduled-tasks"
|
||||
|
||||
tasks = requests.get(f"{base}", params={"limit": 100, "search": "舆情"}).json()
|
||||
for task in tasks["tasks"]:
|
||||
runs = requests.get(f"{base}/{task['task_id']}/runs", params={"limit": 5}).json()
|
||||
for run in runs["runs"]:
|
||||
if run.get("result"):
|
||||
print(task["name"], run["id"], run["result"]["title"])
|
||||
print(run["result"]["content"][:200])
|
||||
```
|
||||
|
||||
### 直接嵌入 iframe
|
||||
|
||||
```html
|
||||
<iframe src="https://<domain>/#/public/scheduled-tasks/<task_id>"
|
||||
style="width:100%;height:800px;border:0"></iframe>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 状态码
|
||||
|
||||
| 状态码 | 含义 |
|
||||
|---|---|
|
||||
| 200 | 成功 |
|
||||
| 400 | 参数错误(如 `limit` 超出范围) |
|
||||
| 404 | 任务 / 执行 / 文件 不存在 |
|
||||
| 5xx | 网关 / 后台错误 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 字段映射快查
|
||||
|
||||
### `task` 字段
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|---|---|---|
|
||||
| `task_id` | string | 任务 ID |
|
||||
| `name` | string | 任务名 |
|
||||
| `schedule_text` | string | 人类可读的执行计划 |
|
||||
| `cron_expr` | string | 规范化的 cron 表达式 |
|
||||
| `timezone` | string | 时区 |
|
||||
| `enabled` | bool | 是否启用 |
|
||||
| `next_run_at` | string | null | 下次执行时间(ISO 8601) |
|
||||
| `last_run_at` | string | null | 上次执行时间 |
|
||||
| `last_status` | string | null | 上次执行状态(`succeeded` / `failed` / `running` / `pending` / `…`) |
|
||||
| `published` | bool | 是否被发布到广场 |
|
||||
|
||||
### `run` 字段
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|---|---|---|
|
||||
| `id` | string | 执行 ID |
|
||||
| `task_id` | string | 所属任务 ID |
|
||||
| `scheduled_for` | string | 原定执行时间 |
|
||||
| `started_at` | string | null | 实际开始时间 |
|
||||
| `finished_at` | string | null | 完成 / 失败时间 |
|
||||
| `status` | string | `pending` / `running` / `succeeded` / `failed` |
|
||||
| `error` | string | null | 失败描述 |
|
||||
| `retry_count` | int | null | 重试次数 |
|
||||
| `result` | object | null | 执行结果(见下) |
|
||||
|
||||
### `result` 字段
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|---|---|---|
|
||||
| `title` | string | 结果标题 |
|
||||
| `content` | string | 主体 Markdown / 文本 |
|
||||
| `format` | string | `markdown` / `text` / `json` / `mixed` |
|
||||
| `files[]` | array | 附加文件元数据,含 `index / name / mime_type / kind / size` |
|
||||
| `created_at` | string | 结果生成时间 |
|
||||
|
||||
---
|
||||
|
||||
## 6. 实现位置
|
||||
|
||||
- 路由实现:[`app/gateway/routers/public_scheduled_tasks.py`](../app/gateway/routers/public_scheduled_tasks.py)
|
||||
- 公开 URL 白名单:[`app/gateway/auth_middleware.py`](../app/gateway/auth_middleware.py) 中的 `_PUBLIC_PATH_PREFIXES`
|
||||
- 前端公开页面:[`frontend-web/src/pages/PublicScheduledTaskPage.tsx`](../../../frontend-web/src/pages/PublicScheduledTaskPage.tsx)
|
||||
- 路由注册:[`frontend-web/src/App.tsx`](../../../frontend-web/src/App.tsx) `/#/public/scheduled-tasks/:taskId`
|
||||
@@ -7,7 +7,8 @@
|
||||
"env": ".env",
|
||||
"graphs": {
|
||||
"lead_agent": "deerflow.agents:make_lead_agent",
|
||||
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent"
|
||||
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent",
|
||||
"ai_writing": "deerflow.agents.ai_writing.graph:make_ai_writing_graph"
|
||||
},
|
||||
"auth": {
|
||||
"path": "./app/gateway/langgraph_auth.py:auth"
|
||||
|
||||
+2
-2
@@ -1,3 +1,3 @@
|
||||
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
|
||||
from .graph import make_ai_writing_graph
|
||||
|
||||
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
|
||||
__all__ = ["make_ai_writing_graph"]
|
||||
+178
-34
@@ -1,7 +1,9 @@
|
||||
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
import logging
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.graph import END, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
@@ -12,6 +14,32 @@ from .nodes.writer_outline import writer_outline_node
|
||||
from .nodes.writer_draft import writer_draft_node
|
||||
from .nodes.editor import editor_node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _mark_session_done(config: RunnableConfig | None) -> None:
|
||||
"""把业务表行 status 写成 ``done``。
|
||||
|
||||
pause_draft_confirm action=finalize / pause_review_confirm force_finalize 时调,
|
||||
让 LangGraph Server 路径下新会话有正确终态(老路径靠 ``_stream_graph_segment``
|
||||
finally 块写终态,新路径完全不走它)。失败不影响 graph 流程。
|
||||
"""
|
||||
if not config:
|
||||
return
|
||||
thread_id = (config.get("configurable") or {}).get("thread_id")
|
||||
if not thread_id:
|
||||
return
|
||||
from deerflow.persistence.ai_writing_sessions import get_default_store
|
||||
from deerflow.runtime.user_context import AUTO
|
||||
|
||||
repo = get_default_store()
|
||||
if repo is None:
|
||||
return
|
||||
try:
|
||||
await repo.update(thread_id, user_id=AUTO, status="done")
|
||||
except Exception:
|
||||
logger.exception("ai_writing: mark_session_done failed (non-fatal) thread_id=%s", thread_id)
|
||||
|
||||
# ── 干预节点 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
@@ -61,6 +89,26 @@ def _normalize_outline(outline: dict) -> dict:
|
||||
return {"title": outline.get("title", ""), "sections": sections}
|
||||
|
||||
|
||||
def _check_outline_material_support(outline: dict) -> str:
|
||||
"""轻量启发式:统计未关联任何素材的章节,返回预警文案(空串=无需预警)。
|
||||
|
||||
仅做提示、不拦截流程;真正的素材不足会在写作节点被检出并触发求助暂停。
|
||||
"""
|
||||
sections = (outline or {}).get("sections", [])
|
||||
weak = [
|
||||
(s.get("section_title") or s.get("sectionTitle") or "")
|
||||
for s in sections
|
||||
if not (s.get("material_ids") or s.get("materialIds"))
|
||||
]
|
||||
weak = [t for t in weak if t]
|
||||
if not weak:
|
||||
return ""
|
||||
return (
|
||||
f"有 {len(weak)} 个章节未关联检索素材({('、'.join(weak))})。"
|
||||
"严格依据素材模式下,这些章节在写作时可能因素材不足而暂停向你求助。"
|
||||
)
|
||||
|
||||
|
||||
async def pause_outline_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-2:大纲确认"""
|
||||
user_input: dict = interrupt({
|
||||
@@ -74,23 +122,37 @@ async def pause_outline_confirm(state: AIWritingState) -> dict:
|
||||
if action == "confirm":
|
||||
raw_outline = user_input.get("editedOutline") or state["current_outline"]
|
||||
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
|
||||
events: list[dict] = [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}]
|
||||
# 阶段四:严格模式下对大纲做轻量素材充分性预警(仅提示,不拦截流程)
|
||||
if state.get("writing_mode") == "strict":
|
||||
warning = _check_outline_material_support(edited)
|
||||
if warning:
|
||||
events.append({
|
||||
"type": "notice", "agent_name": "系统",
|
||||
"title": "素材充分性提示", "detail": warning,
|
||||
})
|
||||
return {
|
||||
"current_outline": edited,
|
||||
"last_user_intervention": user_input,
|
||||
"status": "writing_draft",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}],
|
||||
"progress_events": events,
|
||||
}
|
||||
|
||||
# re_outline
|
||||
feedback = user_input.get("outlineFeedback", "")
|
||||
return {
|
||||
updates: dict = {
|
||||
"last_user_intervention": {**user_input, "_feedback": feedback},
|
||||
"status": "writing_outline",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
|
||||
}
|
||||
# 用户在卡片里手动改过大纲 → 以手动版作为本次重规划的编辑基准
|
||||
raw_edited = user_input.get("editedOutline")
|
||||
if raw_edited:
|
||||
updates["current_outline"] = _normalize_outline(raw_edited)
|
||||
return updates
|
||||
|
||||
|
||||
async def pause_draft_confirm(state: AIWritingState) -> dict:
|
||||
async def pause_draft_confirm(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
"""PAUSE-3:草稿确认"""
|
||||
draft = state["current_draft"]
|
||||
user_input: dict = interrupt({
|
||||
@@ -104,6 +166,7 @@ async def pause_draft_confirm(state: AIWritingState) -> dict:
|
||||
action = user_input.get("action", "to_editor")
|
||||
|
||||
if action == "finalize":
|
||||
await _mark_session_done(config)
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
@@ -127,7 +190,66 @@ async def pause_draft_confirm(state: AIWritingState) -> dict:
|
||||
}
|
||||
|
||||
|
||||
async def pause_review_confirm(state: AIWritingState) -> dict:
|
||||
async def pause_section_help(state: AIWritingState) -> dict:
|
||||
"""素材不足求助点:严格模式下作家发现章节缺素材,让用户逐章决定如何处理。
|
||||
|
||||
用户对每个被阻塞章节选择:
|
||||
- supplement:补充检索素材(整体回到检索流程重走)
|
||||
- delete:删除该章节
|
||||
- loose:按通用知识续写该章节
|
||||
"""
|
||||
blocked = state.get("blocked_sections") or []
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "section_help",
|
||||
"message": "部分章节缺少足够素材支撑写作,请决定如何处理。",
|
||||
"blocked_sections": blocked,
|
||||
})
|
||||
|
||||
# sectionDecisions: [{"sectionTitle": "...", "decision": "supplement|delete|loose"}]
|
||||
by_title: dict = {}
|
||||
for d in user_input.get("sectionDecisions") or []:
|
||||
title = d.get("sectionTitle") or d.get("section_title")
|
||||
if title:
|
||||
by_title[title] = d.get("decision", "loose")
|
||||
|
||||
# 任一章节选「补充检索」→ 整体回到检索,重走「检索 → 素材确认 → 大纲 → 写作」
|
||||
supplement_reasons = [
|
||||
b["reason"] for b in blocked if by_title.get(b["section_title"]) == "supplement"
|
||||
]
|
||||
if supplement_reasons:
|
||||
return {
|
||||
# supplement_search:补充检索(保留已有素材、追加新素材),完成后直接回写作、不重排大纲
|
||||
"last_user_intervention": {**user_input, "action": "supplement_search", "_extra_keywords": supplement_reasons},
|
||||
"status": "researching",
|
||||
"blocked_sections": None,
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "补充检索素材中…"}],
|
||||
}
|
||||
|
||||
# 否则:删除标记章节、记录「通用知识续写」章节,直接回写作
|
||||
outline = state.get("current_outline") or {"title": "", "sections": []}
|
||||
kept, loose_titles = [], []
|
||||
for s in outline.get("sections", []):
|
||||
title = s.get("section_title") or s.get("sectionTitle", "")
|
||||
decision = by_title.get(title)
|
||||
if decision == "delete":
|
||||
continue
|
||||
if decision == "loose":
|
||||
loose_titles.append(title)
|
||||
kept.append(s)
|
||||
# 防御:用户把所有章节都删了 → 保留原大纲并全部按通用知识写,避免空文章
|
||||
if not kept:
|
||||
kept = outline.get("sections", [])
|
||||
loose_titles = [s.get("section_title") or s.get("sectionTitle", "") for s in kept]
|
||||
return {
|
||||
"current_outline": {**outline, "sections": kept},
|
||||
"last_user_intervention": {**user_input, "_loose_sections": loose_titles},
|
||||
"status": "writing_draft",
|
||||
"blocked_sections": None,
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "已按你的选择继续写作…"}],
|
||||
}
|
||||
|
||||
|
||||
async def pause_review_confirm(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
"""PAUSE-4:编辑打回确认"""
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "review_confirm",
|
||||
@@ -138,6 +260,7 @@ async def pause_review_confirm(state: AIWritingState) -> dict:
|
||||
})
|
||||
|
||||
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
|
||||
await _mark_session_done(config)
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
@@ -156,6 +279,11 @@ async def pause_review_confirm(state: AIWritingState) -> dict:
|
||||
|
||||
# ── 条件路由 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def after_researcher(state: AIWritingState) -> str:
|
||||
"""检索完成后:补充检索(素材不足求助)直接回写作,否则去素材确认。"""
|
||||
return "writer_draft" if state.get("status") == "writing_draft" else "pause_material"
|
||||
|
||||
|
||||
def after_material_pause(state: AIWritingState) -> str:
|
||||
return "researcher" if state["status"] == "researching" else "writer_outline"
|
||||
|
||||
@@ -164,6 +292,16 @@ def after_outline_pause(state: AIWritingState) -> str:
|
||||
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
|
||||
|
||||
|
||||
def after_writer_draft(state: AIWritingState) -> str:
|
||||
"""写作完成后:素材不足则去求助暂停点,否则去草稿确认。"""
|
||||
return "pause_section_help" if state["status"] == "awaiting_section_help" else "pause_draft"
|
||||
|
||||
|
||||
def after_section_help_pause(state: AIWritingState) -> str:
|
||||
"""求助暂停点之后:补充检索回 researcher,否则回写作。"""
|
||||
return "researcher" if state["status"] == "researching" else "writer_draft"
|
||||
|
||||
|
||||
def after_draft_pause(state: AIWritingState) -> str:
|
||||
status = state["status"]
|
||||
if status == "done":
|
||||
@@ -188,44 +326,50 @@ def after_review_pause(state: AIWritingState) -> str:
|
||||
return "end" if state["status"] == "done" else "writer_draft"
|
||||
|
||||
|
||||
# ── 组装 Graph ────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_ai_writing_graph():
|
||||
# ── LangGraph Server 入口 ────────────────────────────────────────────────────
|
||||
#
|
||||
# compile 出裸 graph(不带 checkpointer),由 LangGraph Server 注入主 checkpointer
|
||||
# (langgraph.json 里 ``make_checkpointer`` 配的那个)。AI 写作的 thread 状态与
|
||||
# 主聊天共享同一份 saver —— LangGraph schema 用 ``(thread_id, checkpoint_ns)`` 做
|
||||
# 主键,UUID 不会撞。
|
||||
def make_ai_writing_graph(config=None):
|
||||
"""LangGraph Server graph factory:不接收 checkpointer,由 Server 注入。
|
||||
|
||||
Args:
|
||||
config: LangGraph Server 调用工厂时注入的 RunnableConfig(暂未消费)。
|
||||
"""
|
||||
g = StateGraph(AIWritingState)
|
||||
|
||||
g.add_node("intent_parser", intent_parser_node)
|
||||
g.add_node("researcher", researcher_node)
|
||||
g.add_node("pause_material", pause_material_confirm)
|
||||
g.add_node("writer_outline", writer_outline_node)
|
||||
g.add_node("pause_outline", pause_outline_confirm)
|
||||
g.add_node("writer_draft", writer_draft_node)
|
||||
g.add_node("pause_draft", pause_draft_confirm)
|
||||
g.add_node("editor", editor_node)
|
||||
g.add_node("pause_review", pause_review_confirm)
|
||||
g.add_node("intent_parser", intent_parser_node)
|
||||
g.add_node("researcher", researcher_node)
|
||||
g.add_node("pause_material", pause_material_confirm)
|
||||
g.add_node("writer_outline", writer_outline_node)
|
||||
g.add_node("pause_outline", pause_outline_confirm)
|
||||
g.add_node("writer_draft", writer_draft_node)
|
||||
g.add_node("pause_section_help", pause_section_help)
|
||||
g.add_node("pause_draft", pause_draft_confirm)
|
||||
g.add_node("editor", editor_node)
|
||||
g.add_node("pause_review", pause_review_confirm)
|
||||
|
||||
g.set_entry_point("intent_parser")
|
||||
g.add_edge("intent_parser", "researcher")
|
||||
g.add_edge("researcher", "pause_material")
|
||||
|
||||
g.add_edge("intent_parser", "researcher")
|
||||
g.add_conditional_edges("researcher", after_researcher,
|
||||
{"pause_material": "pause_material", "writer_draft": "writer_draft"})
|
||||
g.add_conditional_edges("pause_material", after_material_pause,
|
||||
{"researcher": "researcher", "writer_outline": "writer_outline"})
|
||||
|
||||
{"researcher": "researcher", "writer_outline": "writer_outline"})
|
||||
g.add_edge("writer_outline", "pause_outline")
|
||||
g.add_conditional_edges("pause_outline", after_outline_pause,
|
||||
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
|
||||
|
||||
g.add_edge("writer_draft", "pause_draft")
|
||||
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
|
||||
g.add_conditional_edges("writer_draft", after_writer_draft,
|
||||
{"pause_section_help": "pause_section_help", "pause_draft": "pause_draft"})
|
||||
g.add_conditional_edges("pause_section_help", after_section_help_pause,
|
||||
{"researcher": "researcher", "writer_draft": "writer_draft"})
|
||||
g.add_conditional_edges("pause_draft", after_draft_pause,
|
||||
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
|
||||
|
||||
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
|
||||
g.add_conditional_edges("editor", after_review,
|
||||
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
|
||||
|
||||
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
|
||||
g.add_conditional_edges("pause_review", after_review_pause,
|
||||
{"end": END, "writer_draft": "writer_draft"})
|
||||
{"end": END, "writer_draft": "writer_draft"})
|
||||
|
||||
checkpointer = MemorySaver()
|
||||
return g.compile(checkpointer=checkpointer)
|
||||
|
||||
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph()
|
||||
return g.compile()
|
||||
+48
@@ -15,6 +15,54 @@ from langchain_core.runnables import RunnableConfig
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_ai_writing_config() -> dict:
|
||||
"""读 ``config.yaml`` 的 ``ai_writing`` 段。
|
||||
|
||||
阶段 7 之前由路由层 ``_make_config`` 把这份 dict 注入到 ``configurable.ai_writing_config``
|
||||
给节点使用。阶段 7 删冗余后 ``_make_config`` 和注入路径全部消失(前端 ``stream.submit``
|
||||
的 config 只传 ``model_name``)—— 节点改成自己直接从 ``get_app_config()`` 读,跨边界
|
||||
更干净,``config.yaml`` 改完也不用重启前端 / 重发起会话。
|
||||
|
||||
返回字段(按使用方):
|
||||
- researcher:``keyword_model`` / ``rank_model`` / ``max_materials`` /
|
||||
``search_provider`` 等
|
||||
- editor:``pass_threshold``
|
||||
"""
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
aw = getattr(get_app_config(), "ai_writing", None)
|
||||
if isinstance(aw, dict):
|
||||
return aw
|
||||
return dict(aw) if aw else {}
|
||||
except Exception:
|
||||
logger.exception("ai_writing: 读取 app config 失败,使用空配置")
|
||||
return {}
|
||||
|
||||
|
||||
def safe_get_stream_writer():
|
||||
"""返回一个 stream writer callable;在没有 LangGraph runtime 上下文时退化成 no-op。
|
||||
|
||||
AI 写作节点用它把流式 chunk 推到 LangGraph 原生 ``custom`` 通道。
|
||||
场景:
|
||||
|
||||
- LangGraph Server 路径(``client.runs.stream`` 订阅了 ``stream_mode=["custom", ...]``)
|
||||
→ 真实 writer,事件透传给 SDK 消费方。
|
||||
- 老 ``/start /stream /resume`` 路径(``graph.astream(stream_mode="updates")``)
|
||||
→ 真实 writer 仍可调用,只是没人订阅 custom 通道,事件被静默丢弃。
|
||||
这是阶段 2「硬切到 get_stream_writer」的预期行为:老前端 EventSource 不再收到
|
||||
token-level chunk,但整体进度卡(draft_ready / review_ready 这些 state 级事件)
|
||||
仍正常 —— 阶段 3 前端切到 useStream 后立刻补齐。
|
||||
- 单测里直接调节点函数(无 LangGraph runtime) → 返回 no-op,调用方零分支。
|
||||
"""
|
||||
try:
|
||||
from langgraph.config import get_stream_writer # 延迟 import,便于单测打补丁
|
||||
|
||||
return get_stream_writer()
|
||||
except Exception:
|
||||
return lambda _payload: None
|
||||
|
||||
|
||||
def get_model(config: RunnableConfig, model_name: str | None = None):
|
||||
"""创建对话模型。
|
||||
|
||||
|
||||
+51
-25
@@ -1,8 +1,8 @@
|
||||
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Callable
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
@@ -11,7 +11,7 @@ from ..state import AIWritingState, ReviewResult, ReviewIssue
|
||||
from ..prompts.editor_prompts import (
|
||||
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
from ._utils import get_ai_writing_config, get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,9 +40,9 @@ def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
|
||||
|
||||
async def _stream_one_review(
|
||||
model, user_prompt: str, category: str,
|
||||
sse_queue: asyncio.Queue | None, session_id: str,
|
||||
writer: Callable[[dict], None],
|
||||
) -> str:
|
||||
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
|
||||
"""流式执行单项审查,逐 chunk 经 LangGraph custom 通道推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
|
||||
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
async for chunk in model.astream([
|
||||
@@ -55,16 +55,42 @@ async def _stream_one_review(
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("review_chunk", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
writer({
|
||||
"type": "review_chunk",
|
||||
"category": category,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
})
|
||||
return full_text
|
||||
|
||||
|
||||
async def _save_review_to_repo(config: RunnableConfig, review: ReviewResult) -> None:
|
||||
"""把 review_result 异步落到业务表(best-effort,写失败不影响流程)。
|
||||
|
||||
``user_id`` 走 ``resolve_user_id(AUTO)`` 读 contextvar(与 intent_parser 建行、
|
||||
writer_draft 写草稿同套规则)。
|
||||
"""
|
||||
cfg = config.get("configurable", {}) or {}
|
||||
thread_id = cfg.get("thread_id")
|
||||
if not thread_id:
|
||||
return
|
||||
from deerflow.persistence.ai_writing_sessions import get_default_store
|
||||
from deerflow.runtime.user_context import AUTO
|
||||
|
||||
repo = get_default_store()
|
||||
if repo is None:
|
||||
return
|
||||
try:
|
||||
await repo.update(
|
||||
thread_id,
|
||||
user_id=AUTO,
|
||||
status="review_ready",
|
||||
review_result=dict(review),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("editor: save_review failed (non-fatal) thread_id=%s", thread_id)
|
||||
|
||||
|
||||
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
draft = state.get("current_draft")
|
||||
if not draft:
|
||||
@@ -78,9 +104,9 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
mats = _materials_summary(pkg)
|
||||
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
writer = safe_get_stream_writer()
|
||||
# 阶段 7 后从 app config 直接读,不走 configurable 注入
|
||||
ai_writing_config = get_ai_writing_config()
|
||||
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
|
||||
|
||||
REVIEW_TASKS = [
|
||||
@@ -110,7 +136,7 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
for category, user_prompt in REVIEW_TASKS:
|
||||
fallback = {"score": 80, "issues": []}
|
||||
try:
|
||||
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
|
||||
full_text = await _stream_one_review(model, user_prompt, category, writer)
|
||||
res = parse_json(full_text, fallback)
|
||||
except Exception as e:
|
||||
logger.warning("editor review [%s] failed: %s", category, e)
|
||||
@@ -120,16 +146,13 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
results[category] = res
|
||||
|
||||
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
|
||||
if sse_queue is not None:
|
||||
issues = _parse_issues(res.get("issues", []), category)
|
||||
await sse_queue.put(("review_item_done", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"score": int(res.get("score", 80)),
|
||||
"issues": [dict(iss) for iss in issues],
|
||||
}))
|
||||
else:
|
||||
logger.debug("editor [%s] done (no sse_queue)", category)
|
||||
issues = _parse_issues(res.get("issues", []), category)
|
||||
writer({
|
||||
"type": "review_item_done",
|
||||
"category": category,
|
||||
"score": int(res.get("score", 80)),
|
||||
"issues": [dict(iss) for iss in issues],
|
||||
})
|
||||
|
||||
# 汇总
|
||||
fact_score = int(results["fact"].get("score", 80))
|
||||
@@ -156,6 +179,9 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
revision_notes=all_issues,
|
||||
)
|
||||
|
||||
# 阶段 2:节点内并存 DB 写入(路由层同样也写一遍,阶段 7 删冗余时移除路由那份)
|
||||
await _save_review_to_repo(config, review)
|
||||
|
||||
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
|
||||
notices: list[dict] = []
|
||||
if review_errors:
|
||||
|
||||
+118
-25
@@ -1,12 +1,64 @@
|
||||
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from ..state import AIWritingState
|
||||
from ._utils import llm_json
|
||||
from ._utils import (
|
||||
ThinkTagParser,
|
||||
_content_to_text,
|
||||
get_model,
|
||||
parse_json,
|
||||
safe_get_stream_writer,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _truncate_title(text: str, max_len: int = 50) -> str:
|
||||
"""与老 /start 路由里 _truncate_title 行为一致:UTF 字符截断 + 省略号。"""
|
||||
text = (text or "").strip()
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
return text[:max_len].rstrip() + "…"
|
||||
|
||||
|
||||
async def _upsert_business_row(configurable: dict, state: AIWritingState) -> None:
|
||||
"""在 intent_parser 入口幂等创建 ai_writing_sessions 业务表行。
|
||||
|
||||
LangGraph Server 路径下没有 /start 路由建行,必须节点自建,否则列表页查不到、
|
||||
transcript / draft / review 后续写入都会因 update 找不到行而静默丢数据。
|
||||
resume 路径下 thread_id 已经有行,upsert_session 只刷新 updated_at。
|
||||
|
||||
user_id 走 ``resolve_user_id(AUTO)``:FastAPI 请求线程把它写到 contextvar,
|
||||
asyncio.create_task 继承父任务的 context,所以这里能拿到当前用户;
|
||||
contextvar 没设时(测试 / 离线场景)抛 RuntimeError,try/except 兜底跳过。
|
||||
"""
|
||||
from deerflow.persistence.ai_writing_sessions import get_default_store
|
||||
from deerflow.runtime.user_context import AUTO
|
||||
|
||||
thread_id = configurable.get("thread_id")
|
||||
if not thread_id:
|
||||
return
|
||||
repo = get_default_store()
|
||||
if repo is None:
|
||||
return
|
||||
user_intent = (state.get("user_intent") or "").strip()
|
||||
try:
|
||||
await repo.upsert_session(
|
||||
id=thread_id,
|
||||
title=_truncate_title(user_intent) or "AI 写作",
|
||||
user_intent=user_intent,
|
||||
user_id=AUTO,
|
||||
status="in_progress",
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("intent_parser: upsert_session failed (non-fatal) thread_id=%s", thread_id)
|
||||
|
||||
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
|
||||
_DEFAULT_ARTICLE_TYPES: list[dict] = [
|
||||
@@ -79,6 +131,10 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
|
||||
valid_keys: list[str] = [t["key"] for t in valid_article_types]
|
||||
default_key: str = valid_keys[0] if valid_keys else "analysis"
|
||||
|
||||
# 业务表 upsert(LangGraph Server 路径下没有 /start 路由建行,节点入口自建;
|
||||
# resume 时幂等不重复建。失败不阻断写作流程)。
|
||||
await _upsert_business_row(configurable, state)
|
||||
|
||||
type_options = _build_type_options(valid_article_types)
|
||||
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
|
||||
user = _USER_TEMPLATE.format(
|
||||
@@ -90,42 +146,79 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
|
||||
if fallback_type not in valid_keys:
|
||||
fallback_type = default_key
|
||||
|
||||
result = await llm_json(
|
||||
system,
|
||||
user,
|
||||
fallback={
|
||||
"article_type": fallback_type,
|
||||
"target_audience": state.get("target_audience", "普通读者"),
|
||||
"word_count_target": state.get("word_count_target", 800),
|
||||
"refined_intent": state["user_intent"],
|
||||
},
|
||||
config=config,
|
||||
)
|
||||
fallback_result = {
|
||||
"article_type": fallback_type,
|
||||
"target_audience": state.get("target_audience", "普通读者"),
|
||||
"word_count_target": state.get("word_count_target", 800),
|
||||
"refined_intent": state["user_intent"],
|
||||
}
|
||||
|
||||
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
|
||||
# 流程仍可继续,但用户需知道结构化配置可能不准)。
|
||||
# 流式调用:与 writer_outline / writer_draft 同套机制 —— LLM 逐字吐出 JSON,
|
||||
# 通过 ``intent_chunk`` 推给前端打字机;完成后 ``parse_json`` 解析结构化结果,
|
||||
# 再通过 ``intent_ready`` 推一份结构化字段给前端结果卡片。
|
||||
writer = safe_get_stream_writer()
|
||||
llm_error: str | None = None
|
||||
full_text = ""
|
||||
parser = ThinkTagParser()
|
||||
try:
|
||||
model = get_model(config)
|
||||
async for chunk in model.astream(
|
||||
[SystemMessage(content=system), HumanMessage(content=user)]
|
||||
):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
writer({
|
||||
"type": "intent_chunk",
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
})
|
||||
except Exception as exc:
|
||||
# 模型 / 网络挂掉 —— 不阻断流程,用 fallback 继续。
|
||||
logger.error("intent_parser streaming failed: %s", exc, exc_info=True)
|
||||
llm_error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
result = parse_json(full_text, fallback_result) if full_text else fallback_result
|
||||
|
||||
# 意图解析模型调用失败时以非致命 notice 提示用户(fallback 已兜底,流程仍可继续)
|
||||
progress_events: list[dict] = []
|
||||
if result.get("_llm_error"):
|
||||
if llm_error:
|
||||
progress_events.append({
|
||||
"type": "notice",
|
||||
"agent_name": "系统",
|
||||
"title": "模型错误",
|
||||
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
|
||||
"detail": f"意图解析失败,已按默认配置兜底。原因:{llm_error}",
|
||||
})
|
||||
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
|
||||
|
||||
updates: dict = {"status": "researching", "progress_events": progress_events}
|
||||
|
||||
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
|
||||
raw_type = result.get("article_type", "")
|
||||
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
|
||||
|
||||
if result.get("target_audience"):
|
||||
updates["target_audience"] = result["target_audience"]
|
||||
final_article_type = raw_type if raw_type in valid_keys else fallback_type
|
||||
final_target_audience = result.get("target_audience") or fallback_result["target_audience"]
|
||||
word_count = _coerce_word_count(result.get("word_count_target"))
|
||||
final_word_count = word_count if word_count is not None else fallback_result["word_count_target"]
|
||||
final_refined_intent = result.get("refined_intent") or state["user_intent"]
|
||||
|
||||
# 推送结构化结果给前端 —— 结果卡片显示文章类型 / 目标读者 / 字数 / 精炼意图
|
||||
writer({
|
||||
"type": "intent_ready",
|
||||
"article_type": final_article_type,
|
||||
"target_audience": final_target_audience,
|
||||
"word_count_target": final_word_count,
|
||||
"refined_intent": final_refined_intent,
|
||||
})
|
||||
|
||||
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
|
||||
|
||||
updates: dict = {"status": "researching", "progress_events": progress_events}
|
||||
updates["article_type"] = final_article_type
|
||||
if result.get("target_audience"):
|
||||
updates["target_audience"] = final_target_audience
|
||||
if word_count is not None:
|
||||
updates["word_count_target"] = word_count
|
||||
updates["word_count_target"] = final_word_count
|
||||
if result.get("refined_intent"):
|
||||
updates["user_intent"] = result["refined_intent"]
|
||||
updates["user_intent"] = final_refined_intent
|
||||
|
||||
return updates
|
||||
+144
-43
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
@@ -14,11 +15,66 @@ from ..prompts.researcher_prompts import (
|
||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
||||
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
|
||||
)
|
||||
from ._utils import llm_json
|
||||
from ._utils import get_ai_writing_config, llm_json, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_MATERIALS = 12
|
||||
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 50
|
||||
_DEFAULT_MAX_MATERIALS = 50
|
||||
_MIN_MAX_MATERIALS = 1
|
||||
_MAX_MAX_MATERIALS = 100
|
||||
|
||||
# 素材检索关键词数量边界:用户可指定 1~10 个,非法值回退默认 6
|
||||
_DEFAULT_KEYWORD_COUNT = 6
|
||||
_MAX_KEYWORD_COUNT = 10
|
||||
|
||||
|
||||
def _resolve_keyword_count(raw: object) -> int:
|
||||
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
|
||||
try:
|
||||
n = int(raw) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_KEYWORD_COUNT
|
||||
return max(1, min(_MAX_KEYWORD_COUNT, n))
|
||||
|
||||
|
||||
def _resolve_max_materials(raw: object) -> int:
|
||||
"""把 config.yaml 配置的素材总量上限规整到 [1, 100],非法值回退默认 20。"""
|
||||
try:
|
||||
n = int(raw) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_MAX_MATERIALS
|
||||
return max(_MIN_MAX_MATERIALS, min(_MAX_MAX_MATERIALS, n))
|
||||
|
||||
|
||||
# 占位/示意性关键词(模型误抄提示词模板时产生),检索前剔除
|
||||
_PLACEHOLDER_KEYWORD_RE = re.compile(r"^(关键词\d*|keyword\d*|kw\d*|\.+|…+)$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _clean_keywords(raw: object, fallback: str) -> list[str]:
|
||||
"""清洗 LLM 产出的关键词:去空白、丢弃占位符(如「关键词1」「...」)、去重。
|
||||
|
||||
全部被滤掉时回退为写作意图本身,避免拿占位词检索出一堆无关素材。
|
||||
"""
|
||||
cleaned: list[str] = []
|
||||
for kw in raw if isinstance(raw, list) else []:
|
||||
s = str(kw).strip()
|
||||
if not s or _PLACEHOLDER_KEYWORD_RE.match(s):
|
||||
continue
|
||||
cleaned.append(s)
|
||||
deduped = list(dict.fromkeys(cleaned))
|
||||
return deduped or [fallback]
|
||||
|
||||
|
||||
def _per_keyword_limit(max_materials: int, total_keywords: int) -> int:
|
||||
"""把素材总量上限均分到每个检索词上的检索条数(向上取整)。
|
||||
|
||||
向上取整保证合并去重后素材数仍能接近总量上限;最终会再统一截断到上限。
|
||||
total_keywords <= 0 时回退为总量上限本身。
|
||||
"""
|
||||
if total_keywords <= 0:
|
||||
return max_materials
|
||||
return (max_materials + total_keywords - 1) // total_keywords
|
||||
|
||||
|
||||
def _notice(title: str, detail: str, raw: str = "") -> dict:
|
||||
@@ -37,52 +93,80 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
article_type = state.get("article_type", "analysis")
|
||||
target_audience = state.get("target_audience", "普通读者")
|
||||
|
||||
# 检查是否是补充搜索(PAUSE-1 re_search action)
|
||||
# 检查干预模式(PAUSE-1 素材确认 / 素材不足求助)
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
|
||||
action = intervention.get("action")
|
||||
|
||||
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
|
||||
# 素材不足求助点触发的「补充检索」:保留原大纲,检索完直接回写作、不走素材确认/大纲
|
||||
is_supplement = action == "supplement_search"
|
||||
# PAUSE-1 素材确认卡用户点「追加检索」并填了新关键词:
|
||||
# - 跳过 LLM 重新生成检索词(节省一次调用 + 尊重用户意图)
|
||||
# - 只检索 extra_keywords 这几个新词,不重检索之前 LLM 出过的词
|
||||
# - 保留 existing material_package,按 URL 去重合并
|
||||
is_append_only = action == "re_search" and bool(extra_keywords)
|
||||
|
||||
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
|
||||
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
|
||||
|
||||
# 实时进度推送:阶段 2 起统一走 LangGraph 原生 custom 通道。
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue = cfg.get("sse_queue")
|
||||
sse_session_id: str = cfg.get("sse_session_id", "")
|
||||
writer = safe_get_stream_writer()
|
||||
|
||||
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
|
||||
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
# 注:阶段 7 后这份配置由节点直接读 app config,不再走 configurable 注入。
|
||||
ai_writing_config = get_ai_writing_config()
|
||||
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
|
||||
rank_model: str | None = ai_writing_config.get("rank_model") or None
|
||||
# 素材总量上限(config.yaml: ai_writing.max_materials),按检索词数均分到每个词
|
||||
max_materials = _resolve_max_materials(ai_writing_config.get("max_materials"))
|
||||
|
||||
async def _emit(message: str) -> None:
|
||||
if sse_queue is None:
|
||||
return
|
||||
await sse_queue.put(("research_progress", {
|
||||
"session_id": sse_session_id,
|
||||
"message": message,
|
||||
}))
|
||||
def _emit(message: str) -> None:
|
||||
writer({"type": "research_progress", "message": message})
|
||||
|
||||
# Step 1: 提取关键词
|
||||
await _emit("正在生成检索词…")
|
||||
kw_result = await llm_json(
|
||||
EXTRACT_KEYWORDS_SYSTEM,
|
||||
EXTRACT_KEYWORDS_USER.format(
|
||||
user_intent=intent,
|
||||
article_type=article_type,
|
||||
target_audience=target_audience,
|
||||
),
|
||||
fallback={"keywords": [intent]},
|
||||
config=config,
|
||||
model_name=keyword_model,
|
||||
)
|
||||
keywords: list[str] = kw_result.get("keywords", [intent])
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
kw_result: dict = {} # 走追加分支时跳过 LLM 调用,但下方 notice 拼接需要兜底值
|
||||
if is_append_only:
|
||||
# 追加模式:直接用用户填的关键词,不再调 LLM
|
||||
keywords: list[str] = _clean_keywords(extra_keywords, intent)
|
||||
_emit(f"追加检索 {len(keywords)} 个关键词…")
|
||||
else:
|
||||
_emit("正在生成检索词…")
|
||||
kw_result = await llm_json(
|
||||
EXTRACT_KEYWORDS_SYSTEM,
|
||||
EXTRACT_KEYWORDS_USER.format(
|
||||
user_intent=intent,
|
||||
article_type=article_type,
|
||||
target_audience=target_audience,
|
||||
keyword_count=keyword_count,
|
||||
),
|
||||
fallback={"keywords": [intent]},
|
||||
config=config,
|
||||
model_name=keyword_model,
|
||||
)
|
||||
# 清洗关键词:剔除占位符 / 空值,全空时回退写作意图
|
||||
keywords = _clean_keywords(kw_result.get("keywords"), intent)
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 2: 并发搜索
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
search_keywords = keywords[:6]
|
||||
# 按用户指定的关键词数量截断。
|
||||
# 追加模式:keywords 本身就是用户填的,全部检索不截断(用户填 N 个就搜 N 个)。
|
||||
# 其他模式:受 keyword_count 限制,但 extra_keywords 在内时上限放宽到 keywords 总数。
|
||||
if is_append_only:
|
||||
search_keywords = keywords
|
||||
else:
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
_emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
|
||||
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
|
||||
completed = 0
|
||||
@@ -90,22 +174,26 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
nonlocal completed
|
||||
try:
|
||||
return await provider.search(kw, max_results=3)
|
||||
return await provider.search(kw, max_results=per_keyword)
|
||||
finally:
|
||||
completed += 1
|
||||
if completed < total:
|
||||
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
_emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
else:
|
||||
await _emit(f"已完成全部 {total} 次检索")
|
||||
_emit(f"已完成全部 {total} 次检索")
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
|
||||
# 合并去重
|
||||
# - 首次检索 / 普通 re_search(不带 extra_keywords,用户只想 LLM 重出):旧素材丢弃
|
||||
# - 素材不足求助 supplement_search:保留旧素材 + 追加新检索结果
|
||||
# - PAUSE-1 追加检索 is_append_only:保留旧素材 + 仅检索新关键词
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials: list[Material] = []
|
||||
existing_pkg = state.get("material_package")
|
||||
if existing_pkg and intervention.get("action") != "re_search":
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only)
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
@@ -136,9 +224,9 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
|
||||
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
|
||||
await _emit("检索完成,正在筛选排序素材…")
|
||||
_emit("检索完成,正在筛选排序素材…")
|
||||
rank_error = ""
|
||||
if len(raw_materials) > _MAX_MATERIALS:
|
||||
if len(raw_materials) > max_materials:
|
||||
materials_list = "\n".join(
|
||||
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
|
||||
for m in raw_materials
|
||||
@@ -146,7 +234,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
rank_result = await llm_json(
|
||||
RANK_MATERIALS_SYSTEM,
|
||||
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
|
||||
fallback={"ranked_ids": [m["id"] for m in raw_materials[:_MAX_MATERIALS]], "summary": f"共搜索到{len(raw_materials)}条素材"},
|
||||
fallback={"ranked_ids": [m["id"] for m in raw_materials[:max_materials]], "summary": f"共搜索到{len(raw_materials)}条素材"},
|
||||
config=config,
|
||||
model_name=rank_model,
|
||||
)
|
||||
@@ -159,17 +247,23 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
for m in raw_materials:
|
||||
if m["id"] not in sorted_ids_set:
|
||||
sorted_materials.append(m)
|
||||
materials = sorted_materials[:_MAX_MATERIALS]
|
||||
materials = sorted_materials[:max_materials]
|
||||
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
|
||||
else:
|
||||
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:_MAX_MATERIALS]
|
||||
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:max_materials]
|
||||
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
|
||||
|
||||
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
|
||||
if not materials:
|
||||
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
|
||||
|
||||
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
|
||||
# 追加模式下,pkg.keywords 体现到目前为止累积的检索词(旧 + 新追加),
|
||||
# 让前端列表 / 历史回显能看到完整的搜索路径。
|
||||
pkg_keywords = keywords
|
||||
if is_append_only and existing_pkg:
|
||||
old_kws = list(existing_pkg.get("keywords") or [])
|
||||
pkg_keywords = list(dict.fromkeys(old_kws + keywords))
|
||||
pkg = MaterialPackage(keywords=pkg_keywords, materials=materials, summary=summary)
|
||||
|
||||
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
|
||||
notices: list[dict] = []
|
||||
@@ -187,11 +281,18 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
raw="\n\n----------\n\n".join(raw_dumps[:3]),
|
||||
))
|
||||
|
||||
# 进度事件文案:追加模式下让用户看到只搜了哪几个新词
|
||||
step_start_msg = (
|
||||
f"追加检索关键词:{', '.join(keywords[:3])}"
|
||||
if is_append_only
|
||||
else f"搜索关键词:{', '.join(keywords[:3])}"
|
||||
)
|
||||
return {
|
||||
"material_package": pkg,
|
||||
"status": "awaiting_material_confirm",
|
||||
# 补充检索:直接回写作(沿用原大纲);常规检索:进入素材确认暂停点
|
||||
"status": "writing_draft" if is_supplement else "awaiting_material_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
|
||||
{"type": "step_start", "agent_name": "素材收集专家", "message": step_start_msg},
|
||||
*notices,
|
||||
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
|
||||
],
|
||||
|
||||
+140
-25
@@ -1,18 +1,54 @@
|
||||
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, DraftArticle
|
||||
from ..prompts.writer_prompts import DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER
|
||||
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
|
||||
from ..prompts.writer_prompts import (
|
||||
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
|
||||
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
|
||||
)
|
||||
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 章节正文里的 Markdown 标题行(允许前导空白)
|
||||
_HEADING_PREFIX_RE = re.compile(r"^[ \t]*#{1,6}[ \t]+", re.MULTILINE)
|
||||
|
||||
|
||||
def _normalize_section_headings(text: str) -> str:
|
||||
"""把章节正文里所有 Markdown 标题行统一压成三级标题(###)。
|
||||
|
||||
作家被要求只用 ### 作为「要点」小标题;模型偶尔误用 #/##/#### 时在此纠正,
|
||||
保证文档标题层级符合「文章 # → 章节 ## → 要点 ###」,
|
||||
前端目录栏据此才能把要点正确识别为二级标题。
|
||||
"""
|
||||
return _HEADING_PREFIX_RE.sub("### ", text)
|
||||
|
||||
|
||||
# 疑似「拒绝 / 元话语」的强特征短语:模型把对用户说的话当成正文输出时出现,
|
||||
# 这些短语几乎不可能出现在正常文章里,用作安全网识别。
|
||||
_REFUSAL_MARKERS = (
|
||||
"我无法完成", "无法完成此任务", "无法完成该任务", "无法完成这个任务",
|
||||
"无法完成本次", "无法写出符合", "无法撰写这篇", "无法基于现有素材",
|
||||
"无法基于当前素材", "无法基于提供的素材",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_refusal(text: str) -> bool:
|
||||
"""判断一段章节产出是否疑似「拒绝 / 元话语」而非真正的文章正文。
|
||||
|
||||
模型遇到素材与意图不匹配时,可能把「我无法完成此任务」之类对用户说的话
|
||||
当成正文输出。这里用强特征短语做启发式识别(安全网,非强校验)。
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return False
|
||||
return any(marker in text for marker in _REFUSAL_MARKERS)
|
||||
|
||||
|
||||
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
|
||||
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
|
||||
@@ -27,6 +63,35 @@ def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
|
||||
return "\n".join(lines) if lines else "(无相关素材)"
|
||||
|
||||
|
||||
async def _save_draft_to_repo(config: RunnableConfig, draft: DraftArticle) -> None:
|
||||
"""把 current_draft 异步落到业务表(best-effort,写失败不影响流程)。
|
||||
|
||||
``user_id`` 走 ``resolve_user_id(AUTO)`` 读 contextvar(FastAPI 请求线程设置,
|
||||
asyncio.create_task 继承)。intent_parser 在阶段 4 已用同样规则建行;这里 update
|
||||
时也用 AUTO 才能命中 owner filter。
|
||||
"""
|
||||
cfg = config.get("configurable", {}) or {}
|
||||
thread_id = cfg.get("thread_id")
|
||||
if not thread_id:
|
||||
return
|
||||
from deerflow.persistence.ai_writing_sessions import get_default_store
|
||||
from deerflow.runtime.user_context import AUTO
|
||||
|
||||
repo = get_default_store()
|
||||
if repo is None:
|
||||
return
|
||||
try:
|
||||
await repo.update(
|
||||
thread_id,
|
||||
user_id=AUTO,
|
||||
status="draft_ready",
|
||||
draft_title=draft.get("title") or "",
|
||||
draft_markdown=draft.get("full_markdown") or "",
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("writer_draft: save_draft failed (non-fatal) thread_id=%s", thread_id)
|
||||
|
||||
|
||||
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
outline = state.get("current_outline")
|
||||
if not outline:
|
||||
@@ -39,10 +104,14 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
revision_count = state.get("revision_count", 0)
|
||||
|
||||
# SSE 流式推送所需
|
||||
# 写作模式;用户在「素材不足求助」里选「按通用知识续写」的章节强制走宽松
|
||||
writing_mode = state.get("writing_mode", "loose")
|
||||
loose_titles: set[str] = set(intervention.get("_loose_sections") or [])
|
||||
|
||||
# 流式推送:阶段 2 起统一走 LangGraph 原生 custom 通道(get_stream_writer)。
|
||||
# cfg 仍保留以便阶段 4 取 user_id / thread_id 用。
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
writer = safe_get_stream_writer()
|
||||
|
||||
# 修改意见(来自用户或编辑)
|
||||
revision_notes = ""
|
||||
@@ -89,6 +158,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
section_titles: list[str] = []
|
||||
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
|
||||
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
|
||||
blocked_sections: list[dict] = [] # 严格模式下素材不足、无法写作的章节
|
||||
|
||||
for i, section in enumerate(sections):
|
||||
sec_title = _section_title(i, section)
|
||||
@@ -104,6 +174,10 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
for j, s in enumerate(sections)
|
||||
)
|
||||
|
||||
# 本章节是否走严格模式(全局严格 且 未被用户标记为「通用知识续写」)
|
||||
section_is_strict = writing_mode == "strict" and sec_title not in loose_titles
|
||||
mode_section = WRITER_MODE_STRICT if section_is_strict else WRITER_MODE_LOOSE
|
||||
|
||||
user_msg = DRAFT_SECTION_USER.format(
|
||||
article_title=article_title,
|
||||
outline_context=outline_context,
|
||||
@@ -111,6 +185,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
key_points=key_points_str,
|
||||
section_materials=section_materials,
|
||||
revision_section=revision_notes,
|
||||
mode_section=mode_section,
|
||||
target_words=words_per_section,
|
||||
)
|
||||
|
||||
@@ -127,28 +202,45 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
section_buf += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"is_section_done": False,
|
||||
}))
|
||||
writer({
|
||||
"type": "draft_chunk",
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"is_section_done": False,
|
||||
})
|
||||
|
||||
# 章节完成
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": "",
|
||||
"is_section_done": True,
|
||||
}))
|
||||
writer({
|
||||
"type": "draft_chunk",
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": "",
|
||||
"is_section_done": True,
|
||||
})
|
||||
|
||||
cleaned_buf = section_buf.strip()
|
||||
blocked_reason = ""
|
||||
# 严格模式:作家用哨兵声明素材不足,或仍输出了拒绝话术 → 标记为待求助章节
|
||||
if section_is_strict and SECTION_BLOCKED_MARKER in cleaned_buf:
|
||||
blocked_reason = (
|
||||
cleaned_buf.split(SECTION_BLOCKED_MARKER, 1)[1].strip()[:120]
|
||||
or "素材不足以支撑本章节写作"
|
||||
)
|
||||
elif _looks_like_refusal(cleaned_buf):
|
||||
if section_is_strict:
|
||||
blocked_reason = "素材不足以支撑本章节写作"
|
||||
else:
|
||||
# 宽松模式:模型仍拒绝,安全网占位,不打断流程
|
||||
logger.warning("draft section [%s] looks like a refusal, replaced with placeholder", sec_title)
|
||||
section_errors.append(f"「{sec_title}」生成内容疑似拒绝/无关,已占位")
|
||||
cleaned_buf = "(本节内容生成异常,建议重新生成或补充相关素材)"
|
||||
if blocked_reason:
|
||||
blocked_sections.append({"section_title": sec_title, "reason": blocked_reason})
|
||||
cleaned_buf = f"(本节素材不足:{blocked_reason})"
|
||||
section_titles.append(sec_title)
|
||||
section_bufs.append(section_buf.strip())
|
||||
section_bufs.append(cleaned_buf)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
|
||||
@@ -156,8 +248,26 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
section_bufs.append("(内容生成失败,请重试)")
|
||||
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
|
||||
|
||||
# 严格模式下有章节无法基于素材写作 → 暂停求助用户,不产出半成品草稿
|
||||
if blocked_sections:
|
||||
return {
|
||||
"blocked_sections": blocked_sections,
|
||||
"status": "awaiting_section_help",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
|
||||
{
|
||||
"type": "notice", "agent_name": "作家", "title": "素材不足",
|
||||
"detail": f"有 {len(blocked_sections)} 个章节缺少足够素材支撑写作,需要你决定如何处理。",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def _assemble() -> str:
|
||||
body = "\n\n".join(f"## {t}\n\n{b}" for t, b in zip(section_titles, section_bufs))
|
||||
# 章节正文里的标题统一归一到 ###(要点小标题),章节标题用 ##,文章标题用 #
|
||||
body = "\n\n".join(
|
||||
f"## {t}\n\n{_normalize_section_headings(b)}"
|
||||
for t, b in zip(section_titles, section_bufs)
|
||||
)
|
||||
return f"# {article_title}\n\n{body}"
|
||||
|
||||
full_markdown = _assemble()
|
||||
@@ -205,6 +315,11 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
revision_count=revision_count,
|
||||
)
|
||||
|
||||
# 阶段 2:节点内并存 DB 写入(路由层 _stream_graph_segment 同样也会写一遍,
|
||||
# 阶段 7 删冗余时移除路由层那份)。new path (LangGraph Server) 下路由层
|
||||
# 不再介入,节点写就是唯一来源。
|
||||
await _save_draft_to_repo(config, draft)
|
||||
|
||||
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
|
||||
notices: list[dict] = []
|
||||
if section_errors:
|
||||
|
||||
+56
-34
@@ -1,15 +1,17 @@
|
||||
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, Outline, OutlineSection
|
||||
from ..prompts.writer_prompts import OUTLINE_SYSTEM, OUTLINE_USER
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
from ..prompts.writer_prompts import (
|
||||
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,25 +44,42 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
|
||||
feedback_section = ""
|
||||
if intervention.get("action") == "re_outline" and intervention.get("_feedback"):
|
||||
feedback_section = f"\n重新规划要求:{intervention['_feedback']}"
|
||||
feedback = ""
|
||||
if intervention.get("action") == "re_outline":
|
||||
feedback = intervention.get("_feedback") or intervention.get("outline_feedback") or ""
|
||||
elif intervention.get("outline_feedback"):
|
||||
feedback_section = f"\n重新规划要求:{intervention['outline_feedback']}"
|
||||
feedback = intervention.get("outline_feedback") or ""
|
||||
|
||||
user_prompt = OUTLINE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
materials_summary=_materials_summary(pkg),
|
||||
feedback_section=feedback_section,
|
||||
)
|
||||
# 重新规划且已有大纲 → 走「编辑模式」:把当前大纲喂给模型,按用户要求调整,
|
||||
# 而非从零重生成(否则「加一节/删一节」这类指令没有可编辑的基准)。
|
||||
current_outline = state.get("current_outline")
|
||||
is_revise = bool(feedback) and bool(current_outline)
|
||||
|
||||
# SSE 流式推送
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
if is_revise:
|
||||
system_prompt = OUTLINE_REVISE_SYSTEM
|
||||
user_prompt = OUTLINE_REVISE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
current_outline=json.dumps(current_outline, ensure_ascii=False, indent=2),
|
||||
feedback=feedback,
|
||||
materials_summary=_materials_summary(pkg),
|
||||
)
|
||||
else:
|
||||
system_prompt = OUTLINE_SYSTEM
|
||||
feedback_section = f"\n重新规划要求:{feedback}" if feedback else ""
|
||||
user_prompt = OUTLINE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
materials_summary=_materials_summary(pkg),
|
||||
feedback_section=feedback_section,
|
||||
)
|
||||
|
||||
# 流式推送:阶段 2 起统一走 LangGraph 原生 custom 通道(get_stream_writer)。
|
||||
writer = safe_get_stream_writer()
|
||||
|
||||
full_text = "" # 只累积正文(非思考)用于 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
@@ -70,7 +89,7 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
|
||||
model = get_model(config)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=OUTLINE_SYSTEM),
|
||||
SystemMessage(content=system_prompt),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
@@ -79,12 +98,11 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("outline_chunk", {
|
||||
"session_id": session_id,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
writer({
|
||||
"type": "outline_chunk",
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error("outline streaming failed: %s", e, exc_info=True)
|
||||
return {
|
||||
@@ -98,14 +116,18 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
}],
|
||||
}
|
||||
|
||||
fallback = {
|
||||
"title": "未命名文章",
|
||||
"sections": [
|
||||
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
|
||||
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
|
||||
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
|
||||
],
|
||||
}
|
||||
# 编辑模式解析失败时回退为「当前大纲」,避免把用户已有大纲弄丢
|
||||
if is_revise:
|
||||
fallback = current_outline
|
||||
else:
|
||||
fallback = {
|
||||
"title": "未命名文章",
|
||||
"sections": [
|
||||
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
|
||||
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
|
||||
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
|
||||
],
|
||||
}
|
||||
result = parse_json(full_text, fallback)
|
||||
|
||||
outline = _parse_outline(result)
|
||||
|
||||
+15
-4
@@ -1,9 +1,13 @@
|
||||
# 注:两个 USER 提示词末尾的 `/no_think` 是 Qwen3 系列的"软开关",
|
||||
# 收到当轮即跳过思考过程;对其它大模型只是一行普通文本、无副作用。
|
||||
# 中文「严禁思考」强令块对所有模型生效,保留不动。
|
||||
|
||||
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:为写作任务提取搜索关键词。
|
||||
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从写作意图中提取 3-6 个搜索关键词
|
||||
1. 从写作意图中按要求的数量提取搜索关键词
|
||||
你不能写文章,不能评价内容好坏,只做关键词提取。
|
||||
|
||||
【严禁思考】
|
||||
@@ -25,8 +29,13 @@ EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
|
||||
请提取 3-6 个搜索关键词,用于搜集相关素材。只输出 JSON:
|
||||
{{"keywords": ["关键词1", "关键词2"]}}"""
|
||||
请提取 {keyword_count} 个搜索关键词,用于搜集相关素材。
|
||||
关键词数量必须为 {keyword_count} 个,不多不少。
|
||||
关键词必须是依据上述写作意图提炼的真实检索词,不要照抄下方 JSON 模板里的占位符。
|
||||
只输出 JSON(下方 ... 是占位示意,不是真实关键词):
|
||||
{{"keywords": ["...", "..."]}}
|
||||
|
||||
/no_think"""
|
||||
|
||||
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:对搜索结果按相关度排序和筛选。
|
||||
@@ -56,4 +65,6 @@ RANK_MATERIALS_USER = """写作意图:{user_intent}
|
||||
{materials_list}
|
||||
|
||||
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
|
||||
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}"""
|
||||
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}
|
||||
|
||||
/no_think"""
|
||||
+103
-12
@@ -5,7 +5,17 @@ OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你只能做以下事情:
|
||||
1. 分析素材,提取核心观点
|
||||
2. 规划文章章节结构和每节要点
|
||||
你不能写正文,只写大纲。
|
||||
你不能写正文,只写大纲。大纲须紧扣写作意图,并尽量覆盖可用素材中的核心信息。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
|
||||
- 禁止输出任何对用户说的话或对任务的评价(如"素材不足""无法规划大纲");无论素材是否充分,都必须直接产出大纲 JSON。
|
||||
- 收到输入后立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【要点(key_points)的特殊要求】
|
||||
- 每个要点都会作为文章里的一个二级小标题呈现,所以要点必须写成"标题式短语"。
|
||||
- 用简洁的名词性短语,控制在 15 字以内,不要写成完整句子,不要带句末标点。
|
||||
- 每个章节规划 2-4 个要点,不要过多,避免文章被切得过碎。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
@@ -17,9 +27,9 @@ OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
{
|
||||
"title": "新能源汽车出口为何高速增长",
|
||||
"sections": [
|
||||
{"section_title": "引言:出口数据创新高", "key_points": ["今年出口量同比增长", "在全球市场的排名变化"], "material_ids": ["m001"]},
|
||||
{"section_title": "引言:出口数据创新高", "key_points": ["出口量同比增幅", "全球市场排名变化"], "material_ids": ["m001"]},
|
||||
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
|
||||
{"section_title": "结论与展望", "key_points": ["未来增长面临的挑战", "行业趋势判断"], "material_ids": ["m004"]}
|
||||
{"section_title": "结论与展望", "key_points": ["未来增长挑战", "行业趋势判断"], "material_ids": ["m004"]}
|
||||
]
|
||||
}"""
|
||||
|
||||
@@ -33,21 +43,98 @@ OUTLINE_USER = """写作意图:{user_intent}
|
||||
|
||||
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
|
||||
|
||||
# ── 重新规划(基于现有大纲的编辑)─────────────────────────────────────────────
|
||||
|
||||
OUTLINE_REVISE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据用户的修改要求,调整一份【已有的文章大纲】。
|
||||
你的工作背景:你有十年内容创作经验,擅长精准理解修改意图并落实到大纲结构上。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 在「当前大纲」基础上,按「用户修改要求」做调整
|
||||
你不能写正文,只输出调整后的大纲。
|
||||
|
||||
【修改原则】(最高优先级)
|
||||
- 准确识别并落实用户意图,常见操作:新增章节、删除章节、合并章节、拆分章节、调整章节顺序、修改章节标题、调整章节要点。
|
||||
- 只改用户明确提到的部分:用户没要求改动的章节,必须原样保留(标题、要点、material_ids 全部不变)。
|
||||
- 新增章节时:紧扣写作意图补充合理章节,并尽量关联可用素材的 material_ids。
|
||||
- 章节数量完全由用户要求决定,没有上下限——用户要加就加、要减就减。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
|
||||
- 禁止输出任何对用户说的话;收到输入后立即、直接输出最终 JSON。
|
||||
|
||||
【要点(key_points)的特殊要求】
|
||||
- 每个要点会作为文章里的二级小标题呈现,写成简洁名词性短语,15 字以内,不带句末标点。
|
||||
- 每个章节 2-4 个要点。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个调整后的【完整】大纲 JSON 对象,不要解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
|
||||
- 结构与「当前大纲」一致:
|
||||
{"title": "...", "sections": [{"section_title": "...", "key_points": ["..."], "material_ids": ["..."]}]}"""
|
||||
|
||||
OUTLINE_REVISE_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
目标字数:{word_count_target}字
|
||||
|
||||
当前大纲(JSON):
|
||||
{current_outline}
|
||||
|
||||
用户修改要求:{feedback}
|
||||
|
||||
可用素材摘要:
|
||||
{materials_summary}
|
||||
|
||||
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
|
||||
|
||||
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据大纲和素材,写出一段高质量的文章内容。
|
||||
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
|
||||
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 根据给定的章节要点和素材写这一段内容
|
||||
2. 观点必须来自素材,不编造数据
|
||||
2. 优先采用素材中的事实与数据;素材不足时可结合你的通用知识与常识补充论述,但须紧扣写作意图与章节主题,不要脱离主题杜撰具体的数字、人名、机构名与日期
|
||||
你不能修改大纲,不能写其他章节,只写当前这一段。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止"让我想想""首先""我需要""根据写作要求"之类的过渡语。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容。
|
||||
- 禁止输出任何对用户说的话或对任务本身的评价,例如"我无法完成此任务""您提供的素材无法支撑写作""建议您提供……素材""素材与意图不匹配"等——这类话绝不允许出现在文章里。
|
||||
- 无论素材是否充分,你都必须直接产出文章正文本身;收到输入后立即开始写正文(含 ### 小标题),不要有任何前置说明。
|
||||
|
||||
【结构要求】
|
||||
- 把本章节的每一个「章节要点」写成一个二级小标题,用 Markdown 三级标题语法:### 要点
|
||||
- 每个 ### 小标题下面紧跟该要点的论述正文,正文连贯成段。
|
||||
- 即使章节只有一个要点,也要为它写出对应的 ### 小标题。
|
||||
- 小标题文字简洁,直接沿用给定的要点措辞,不要加序号、不要加标点。
|
||||
- 只能使用 ### 这一级标题;绝不要输出 # 或 ##(文章标题、章节标题由系统统一添加)。
|
||||
|
||||
【输出要求】
|
||||
- 直接返回这一段的正文文字,不要 JSON,不要章节标题,不要解释。
|
||||
- 直接返回这一段的内容(含 ### 小标题与正文),不要 JSON,不要解释。
|
||||
- 不要写"以下是正文""本段内容如下"之类的开场白。
|
||||
- 段落要连贯成文,不要罗列要点。
|
||||
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
|
||||
- 充分展开每个要点:每个要点至少用 2-3 句话论述,结合素材中的事实与数据说理,写满要求的篇幅。"""
|
||||
- 充分展开每个要点:每个要点至少用 2-3 句话论述,尽量结合素材中的事实与数据说理,写满要求的篇幅。"""
|
||||
|
||||
# ── 写作模式说明:注入 DRAFT_SECTION_USER 的 {mode_section} ───────────────────
|
||||
|
||||
# 章节无法基于素材写作时,作家在严格模式下输出的哨兵标记
|
||||
SECTION_BLOCKED_MARKER = "[[SECTION_BLOCKED]]"
|
||||
|
||||
WRITER_MODE_LOOSE = """
|
||||
【写作模式:宽松】
|
||||
素材充分时优先采用素材;素材不足时可结合你的通用知识与常识补充论述,
|
||||
但必须紧扣写作意图与本章节主题,不要脱离主题杜撰具体数字、人名、机构名。"""
|
||||
|
||||
WRITER_MODE_STRICT = """
|
||||
【写作模式:严格依据素材】
|
||||
本章节论述必须严格基于上方「相关素材」,不得使用素材之外的知识,不得杜撰任何事实与数据。
|
||||
如果上方素材确实不足以支撑本章节的写作,不要勉强拼凑、不要写无关内容,
|
||||
此时只输出一行:{marker} 一句话说明缺什么素材(例:{marker} 缺少关于XX的具体数据与案例),
|
||||
此外不要输出任何内容。只要素材能支撑哪怕部分要点,就正常写作,不要输出该标记。""".format(
|
||||
marker=SECTION_BLOCKED_MARKER
|
||||
)
|
||||
|
||||
DRAFT_SECTION_USER = """文章标题:{article_title}
|
||||
文章整体结构:
|
||||
@@ -58,8 +145,9 @@ DRAFT_SECTION_USER = """文章标题:{article_title}
|
||||
|
||||
相关素材:
|
||||
{section_materials}
|
||||
{revision_section}
|
||||
请写出「{section_title}」这一章节的正文内容,不少于 {target_words} 字,论述充分、内容充实。"""
|
||||
{revision_section}{mode_section}
|
||||
请写出「{section_title}」这一章节的内容,不少于 {target_words} 字。
|
||||
把上述每个章节要点写成一个 ### 二级小标题,并在标题下展开论述,论述充分、内容充实。"""
|
||||
|
||||
DRAFT_EXPAND_USER = """文章标题:{article_title}
|
||||
当前章节:{section_title}
|
||||
@@ -71,5 +159,8 @@ DRAFT_EXPAND_USER = """文章标题:{article_title}
|
||||
这一章节已有内容如下:
|
||||
{existing_text}
|
||||
|
||||
以上内容篇幅不足。请在保留原意和事实的基础上重写并扩充这一章节,使正文不少于 {target_words} 字。
|
||||
要求:补充更多来自素材的事实、数据与分析,论述更充分、更具体;直接返回扩充后的完整章节正文,不要章节标题、不要解释。"""
|
||||
以上内容篇幅不足。请在保留原意、事实和 ### 二级小标题结构的基础上重写并扩充这一章节,使内容不少于 {target_words} 字。
|
||||
要求:
|
||||
- 补充更多来自素材的事实、数据与分析,论述更充分、更具体。
|
||||
- 保留并沿用原有的 ### 小标题(每个章节要点对应一个),只用 ### 这一级标题,不要输出 # 或 ##。
|
||||
- 直接返回扩充后的完整章节内容(含 ### 小标题),不要解释。"""
|
||||
+14
-1
@@ -73,17 +73,27 @@ class UserInterventionPayload(TypedDict):
|
||||
WritingStatus = Literal[
|
||||
"researching", "awaiting_material_confirm",
|
||||
"writing_outline", "awaiting_outline_confirm",
|
||||
"writing_draft", "awaiting_draft_confirm",
|
||||
"writing_draft", "awaiting_section_help", "awaiting_draft_confirm",
|
||||
"reviewing", "awaiting_review_confirm",
|
||||
"revising", "done", "error",
|
||||
]
|
||||
|
||||
|
||||
class BlockedSection(TypedDict):
|
||||
"""严格模式下素材不足、无法写作的章节。"""
|
||||
section_title: str
|
||||
reason: str
|
||||
|
||||
class AIWritingState(TypedDict):
|
||||
# 写作配置
|
||||
user_intent: str
|
||||
article_type: str
|
||||
target_audience: str
|
||||
word_count_target: int
|
||||
# 素材检索关键词数量(用户指定,1~10,默认 6)
|
||||
keyword_count: int
|
||||
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
|
||||
writing_mode: str
|
||||
|
||||
# Agent A 输出
|
||||
material_package: Optional[MaterialPackage]
|
||||
@@ -99,6 +109,9 @@ class AIWritingState(TypedDict):
|
||||
# 用户干预数据(每个干预点会覆盖)
|
||||
last_user_intervention: Optional[UserInterventionPayload]
|
||||
|
||||
# 严格模式下素材不足、待用户决策的章节(写作节点产出,求助暂停点消费)
|
||||
blocked_sections: Optional[List[BlockedSection]]
|
||||
|
||||
# 流程控制
|
||||
revision_count: int
|
||||
max_revisions: int
|
||||
|
||||
+50
-5
@@ -268,6 +268,7 @@ def _build_middlewares(
|
||||
# Add TodoList middleware if plan mode is enabled
|
||||
cfg = _get_runtime_config(config)
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
|
||||
if todo_list_middleware is not None:
|
||||
middlewares.append(todo_list_middleware)
|
||||
@@ -279,8 +280,10 @@ def _build_middlewares(
|
||||
# Add TitleMiddleware
|
||||
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
|
||||
|
||||
# Add MemoryMiddleware (after TitleMiddleware)
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory))
|
||||
# Add MemoryMiddleware (after TitleMiddleware) — skip in bootstrap mode (agent creation flow)
|
||||
memory_injection_enabled = cfg.get("memory_injection_enabled", True)
|
||||
if not is_bootstrap:
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
|
||||
|
||||
# Add ViewImageMiddleware only if the current model supports vision.
|
||||
# Use the resolved runtime model_name from make_lead_agent to avoid stale config values.
|
||||
@@ -309,6 +312,37 @@ def _build_middlewares(
|
||||
|
||||
middlewares.append(SetupAgentApprovalMiddleware())
|
||||
|
||||
# SkillStopMiddleware — intercepts skill invocations and halts execution
|
||||
# when a configured skill is about to run. Request-level options
|
||||
# (skill_stop_names / skill_stop_message) take precedence; static
|
||||
# config in ``app_config.skill_stop`` is the fallback.
|
||||
skill_stop_names = cfg.get("skill_stop_names")
|
||||
skill_stop_message = cfg.get("skill_stop_message")
|
||||
if skill_stop_names:
|
||||
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
|
||||
|
||||
middlewares.append(
|
||||
SkillStopMiddleware(
|
||||
skill_names=skill_stop_names,
|
||||
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
|
||||
stop_message=skill_stop_message,
|
||||
stop_on_match=True,
|
||||
)
|
||||
)
|
||||
logger.info("SkillStopMiddleware enabled (from request): %s", skill_stop_names)
|
||||
elif resolved_app_config.skill_stop.enabled and resolved_app_config.skill_stop.skill_names:
|
||||
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
|
||||
|
||||
middlewares.append(
|
||||
SkillStopMiddleware(
|
||||
skill_names=resolved_app_config.skill_stop.skill_names,
|
||||
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
|
||||
stop_message=resolved_app_config.skill_stop.stop_message,
|
||||
stop_on_match=resolved_app_config.skill_stop.stop_on_match,
|
||||
)
|
||||
)
|
||||
logger.info("SkillStopMiddleware enabled (from config): %s", resolved_app_config.skill_stop.skill_names)
|
||||
|
||||
# ClarificationMiddleware should always be last
|
||||
middlewares.append(ClarificationMiddleware())
|
||||
return middlewares
|
||||
@@ -336,13 +370,19 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
subagent_enabled = cfg.get("subagent_enabled", False)
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
|
||||
writing_mode = bool(cfg.get("writing_mode", False))
|
||||
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
|
||||
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
|
||||
writing_artifact_path = None
|
||||
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
|
||||
requested_excluded_tools = cfg.get("excluded_tools") or None
|
||||
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
try:
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
except FileNotFoundError:
|
||||
logger.warning("Agent directory not found for '%s'; running with default config", agent_id)
|
||||
agent_config = None
|
||||
agent_display_name = agent_config.name if agent_config else agent_id
|
||||
# Custom agent model from agent config (if any), or None to let _resolve_model_name pick the default
|
||||
agent_model_name = agent_config.model if agent_config and agent_config.model else None
|
||||
@@ -375,7 +415,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
config["metadata"] = {}
|
||||
|
||||
if is_bootstrap:
|
||||
runtime_available_skills: list[str] | None = ["bootstrap"]
|
||||
runtime_available_skills: list[str] | None = None # all enabled skills visible in bootstrap
|
||||
elif agent_config:
|
||||
# Custom agents must use an explicit allowlist. If omitted, default to none.
|
||||
runtime_available_skills = list(agent_config.skills or [])
|
||||
@@ -407,6 +447,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
model_name=model_name,
|
||||
subagent_enabled=subagent_enabled,
|
||||
is_scheduled_run=cfg.get("is_scheduled_run", False),
|
||||
excluded_tools=requested_excluded_tools,
|
||||
app_config=resolved_app_config,
|
||||
)
|
||||
+ [setup_agent],
|
||||
@@ -416,8 +457,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
writing_mode=writing_mode,
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=set(["bootstrap"]),
|
||||
available_skills=None,
|
||||
is_bootstrap=True,
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
@@ -430,6 +473,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
groups=agent_config.tool_groups if agent_config else None,
|
||||
subagent_enabled=subagent_enabled,
|
||||
is_scheduled_run=cfg.get("is_scheduled_run", False),
|
||||
excluded_tools=requested_excluded_tools,
|
||||
app_config=resolved_app_config,
|
||||
),
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
@@ -442,6 +486,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+45
-2
@@ -464,11 +464,22 @@ You are {agent_name}, an internal super agent.
|
||||
- ❌ DO NOT skip clarification for "efficiency" - accuracy matters more than speed
|
||||
- ❌ DO NOT make assumptions when information is missing - ALWAYS ask
|
||||
- ❌ DO NOT proceed with guesses - STOP and call ask_clarification first
|
||||
- ❌ DO NOT cram several unrelated questions into one `question` string — split them into separate tool calls
|
||||
- ✅ Analyze the request in thinking → Identify unclear aspects → Ask BEFORE any action
|
||||
- ✅ If you identify the need for clarification in your thinking, you MUST call the tool IMMEDIATELY
|
||||
- ✅ After calling ask_clarification, execution will be interrupted automatically
|
||||
- ✅ Wait for user response - do NOT continue with assumptions
|
||||
|
||||
**Batching multiple independent questions:**
|
||||
- When the request leaves **several independent aspects** unclear (e.g. project type AND tech stack
|
||||
AND deployment target), call `ask_clarification` multiple times in the same turn — one call per
|
||||
aspect, each with its own focused `question` and `options`.
|
||||
- The frontend groups all clarification cards from one turn into a single submission, so the user
|
||||
answers them together in one reply. You will then receive ONE combined user message addressing
|
||||
every question in order — read it carefully and match answers back to the questions you asked.
|
||||
- If the next question only makes sense after the previous answer (sequential dependency), ask
|
||||
only the first one this turn and wait for the reply before asking the next.
|
||||
|
||||
**How to Use:**
|
||||
```python
|
||||
ask_clarification(
|
||||
@@ -496,6 +507,8 @@ You: "Deploying to staging..." [proceed]
|
||||
|
||||
{skills_section}
|
||||
|
||||
{bootstrap_section}
|
||||
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
@@ -602,7 +615,7 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
|
||||
"""
|
||||
|
||||
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
|
||||
@@ -612,10 +625,13 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
Args:
|
||||
agent_name: agent 标识;``None`` 时按 "default" 处理。
|
||||
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
|
||||
injection_enabled: 运行时开关;``False`` 时跳过注入直接返回空串。
|
||||
|
||||
Returns:
|
||||
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
|
||||
"""
|
||||
if not injection_enabled:
|
||||
return ""
|
||||
try:
|
||||
from deerflow.agents.memory.manager import build_memory_manager
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
@@ -819,6 +835,27 @@ def _build_custom_mounts_section(*, app_config: AppConfig | None = None) -> str:
|
||||
return f"\n**Custom Mounted Directories:**\n{mounts_list}\n- If the user needs files outside `/mnt/user-data`, use these absolute container paths directly when they match the requested directory"
|
||||
|
||||
|
||||
def _build_bootstrap_section() -> str:
|
||||
return """<bootstrap_agent_creation>
|
||||
你正在帮助用户创建一个自定义智能体。
|
||||
|
||||
**技能匹配规则(必须遵守):**
|
||||
1. 先调用 `skill_list` 查看用户当前所有可用技能及其描述。
|
||||
2. 根据用户对新智能体的描述/意图,从已有技能中选出最相关的 1-5 个技能名称(skill name,使用 hyphen-case)。
|
||||
3. 将选出的技能名称填入 `setup_agent` 的 `skills` 参数。
|
||||
4. **严禁在本次创建流程中调用 `skill_manage`(创建、编辑或删除技能)**。如用户提出需要新技能,请告知他们创建完智能体后再单独创建。
|
||||
5. 如果没有任何技能与用户意图相关,可将 `skills` 设为空列表,并向用户说明原因。
|
||||
|
||||
**工作流程:**
|
||||
1. 与用户充分沟通,明确新智能体的用途、专长和边界。
|
||||
2. 调用 `skill_list` 列出现有技能。
|
||||
3. 在思考中匹配技能,选出最合适的。
|
||||
4. 草拟 SOUL.md(定义智能体的人设、工作方式、输出要求)。
|
||||
5. 向用户确认配置后,调用 `setup_agent` 提交。
|
||||
</bootstrap_agent_creation>
|
||||
"""
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
@@ -828,12 +865,14 @@ def apply_prompt_template(
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
available_skills: set[str] | None = None,
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
) -> str:
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
memory_context = _get_memory_context(storage_agent_id, app_config=app_config)
|
||||
memory_context = _get_memory_context(storage_agent_id, app_config=app_config, injection_enabled=memory_injection_enabled)
|
||||
|
||||
# Include subagent section only if enabled (from runtime parameter)
|
||||
n = max_concurrent_subagents
|
||||
@@ -857,6 +896,9 @@ def apply_prompt_template(
|
||||
else ""
|
||||
)
|
||||
|
||||
# Bootstrap section: skill-matching guidance (only in agent creation flow)
|
||||
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
|
||||
|
||||
# Get skills section
|
||||
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
|
||||
|
||||
@@ -902,6 +944,7 @@ Writing mode is enabled for this run.
|
||||
agent_name=agent_name or storage_agent_id or "CM 助手",
|
||||
soul=get_agent_soul(storage_agent_id),
|
||||
skills_section=skills_section,
|
||||
bootstrap_section=bootstrap_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
memory_context=memory_context,
|
||||
|
||||
+42
-21
@@ -223,8 +223,11 @@ def build_memory_manager(
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
|
||||
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
|
||||
这是构建 manager 的统一入口 —— 根据 ``config.version`` 选择 V1 或 V2 provider:
|
||||
|
||||
- ``version="v1"``: 仅注册 V1BuiltinProvider (memory.json + LLM 自动提取)
|
||||
- ``version="v2"``: 注册 BuiltinFileProvider + 可选 HindsightProvider (USER.md/MEMORY.md)
|
||||
|
||||
所有 provider 已 ``initialize`` 完毕,可直接使用。
|
||||
|
||||
Parameters
|
||||
@@ -234,7 +237,6 @@ def build_memory_manager(
|
||||
config:
|
||||
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
|
||||
"""
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
@@ -242,34 +244,53 @@ def build_memory_manager(
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = MemoryManager()
|
||||
base_dir = str(get_paths().base_dir)
|
||||
|
||||
if config.builtin.enabled:
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
v1_cfg = config.v1
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
V1BuiltinProvider(
|
||||
max_facts=v1_cfg.max_facts,
|
||||
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
|
||||
debounce_seconds=v1_cfg.debounce_seconds,
|
||||
model_name=v1_cfg.model_name,
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
if config.builtin.enabled:
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
)
|
||||
)
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id or "default",
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
base_dir=str(get_paths().base_dir),
|
||||
base_dir=base_dir,
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
+39
@@ -69,3 +69,42 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
filtered.append(msg)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
_CORRECTION_PATTERNS = re.compile(
|
||||
r"\b(?:no[,.]?\s+(?:that'?s?\s+)?(?:wrong|incorrect|not\s+right|not\s+what)|"
|
||||
r"actually[,.]?\s+(?:it\s+(?:is|should)|you\s+(?:should|need|must)|the\s+correct)|"
|
||||
r"that'?s?\s+(?:wrong|incorrect|not\s+right|not\s+what\s+i)|"
|
||||
r"you\s+(?:got\s+it\s+wrong|misunderstood|made\s+an?\s+error|made\s+a\s+mistake)|"
|
||||
r"incorrect[,.]|wrong[,.]|mistake[,.]|correction:|that\s+is\s+not\s+right)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_REINFORCEMENT_PATTERNS = re.compile(
|
||||
r"\b(?:perfect[,.]?|exactly[,.]?|correct[,.]?|that'?s?\s+(?:right|correct|perfect|exactly\s+right)|"
|
||||
r"you\s+(?:got\s+it|nailed\s+it)|great\s+(?:job|work)|well\s+done|"
|
||||
r"yes[,.]?\s+(?:that'?s?\s+(?:right|correct|exactly)|exactly)|"
|
||||
r"keep\s+(?:it\s+up|doing\s+that|going)|"
|
||||
r"(?:that'?s?\s+|this\s+is\s+)?(?:exactly|precisely)\s+(?:what\s+i|right))\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def detect_correction(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains explicit user correction signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _CORRECTION_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_reinforcement(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains positive reinforcement signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _REINFORCEMENT_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based MemoryProvider 实现。
|
||||
|
||||
存储格式:``{base_dir}/users/{user_id}/memory.json``(经 FileMemoryStorage 管理)
|
||||
JSON 结构使用嵌套 ``{summary, updatedAt}`` 对象,与原版 DeerFlow 完全一致。
|
||||
|
||||
LLM 在 V1 模式下不调用 memory 工具 —— 事实完全由异步 LLM 提取。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _FakeMsg:
|
||||
"""轻量 message 对象,用于将文本传给原版 format_conversation_for_update。"""
|
||||
__slots__ = ("type", "content")
|
||||
|
||||
def __init__(self, type_: str, content: str) -> None:
|
||||
self.type = type_
|
||||
self.content = content
|
||||
|
||||
|
||||
class V1BuiltinProvider(MemoryProvider):
|
||||
"""原版 DeerFlow V1 记忆 provider。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_facts: int = 100,
|
||||
fact_confidence_threshold: float = 0.7,
|
||||
debounce_seconds: int = 30,
|
||||
model_name: str = "",
|
||||
token_budget: int = 2000,
|
||||
) -> None:
|
||||
self._max_facts = max_facts
|
||||
self._confidence_threshold = fact_confidence_threshold
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._model_name = model_name
|
||||
self._token_budget = token_budget
|
||||
|
||||
self._user_id: str = ""
|
||||
self._thread_id: str = ""
|
||||
self._agent_id: str | None = None
|
||||
self._frozen_block: str = ""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "v1_builtin"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
_ = base_dir # FileMemoryStorage uses get_paths() internally
|
||||
self._user_id = user_id
|
||||
self._thread_id = thread_id or ""
|
||||
self._agent_id = agent_id
|
||||
self._frozen_block = self._build_frozen_block()
|
||||
|
||||
# ---- 系统提示词 -------------------------------------------------------
|
||||
|
||||
def _build_frozen_block(self) -> str:
|
||||
"""从 memory.json 读取数据,用 format_memory_for_injection 渲染注入块。
|
||||
|
||||
当 agent_id 已知时加载 per-agent 记忆;否则加载用户级记忆。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.memory.v1.prompt import format_memory_for_injection
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
|
||||
data = get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
text = format_memory_for_injection(data, max_tokens=self._token_budget)
|
||||
if not text:
|
||||
return ""
|
||||
return f"<memory>\n{text}\n</memory>"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider 加载 memory.json 失败: %s", exc)
|
||||
return ""
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
return self._frozen_block
|
||||
|
||||
# ---- 召回 / 写入 ------------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
# V1 无动态召回,静态 snapshot 已在 system_prompt_block 里
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""把本轮对话加入防抖队列,等待 LLM 异步提取事实。"""
|
||||
if not user_content.strip() and not assistant_content.strip():
|
||||
return
|
||||
try:
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
|
||||
msgs = [
|
||||
_FakeMsg("human", user_content),
|
||||
_FakeMsg("ai", assistant_content),
|
||||
]
|
||||
queue = get_memory_queue()
|
||||
queue.add(
|
||||
thread_id=thread_id or self._thread_id,
|
||||
messages=msgs,
|
||||
user_id=self._user_id,
|
||||
agent_name=self._agent_id or None,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider sync_turn 入队失败: %s", exc)
|
||||
|
||||
# ---- 工具 (V1 不暴露工具给 LLM) -------------------------------------
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
# ---- API 读取接口 (供 Gateway router 用) -----------------------------
|
||||
|
||||
def read_v1_data(self) -> dict[str, Any]:
|
||||
"""读取 memory.json 的当前内容(实时读磁盘,非 frozen snapshot)。
|
||||
|
||||
当 agent_id 已知时读取 per-agent 记忆;否则读用户级记忆。
|
||||
"""
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
return get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
|
||||
def fact_count(self) -> int:
|
||||
try:
|
||||
return len(self.read_v1_data().get("facts", []))
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
|
||||
def usage_text(self) -> str:
|
||||
return f"{self.fact_count()}/{self._max_facts} 个事实"
|
||||
|
||||
|
||||
__all__ = ["V1BuiltinProvider"]
|
||||
+1
@@ -0,0 +1 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based 记忆子模块。"""
|
||||
+371
@@ -0,0 +1,371 @@
|
||||
"""Prompt templates for memory update and injection."""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
TIKTOKEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
TIKTOKEN_AVAILABLE = False
|
||||
|
||||
# Prompt template for updating memory based on conversation
|
||||
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
|
||||
|
||||
Current Memory State:
|
||||
<current_memory>
|
||||
{current_memory}
|
||||
</current_memory>
|
||||
|
||||
New Conversation to Process:
|
||||
<conversation>
|
||||
{conversation}
|
||||
</conversation>
|
||||
|
||||
Instructions:
|
||||
1. Analyze the conversation for important information about the user
|
||||
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
|
||||
3. Update the memory sections as needed following the detailed length guidelines below
|
||||
|
||||
Before extracting facts, perform a structured reflection on the conversation:
|
||||
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
|
||||
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
|
||||
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
|
||||
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
|
||||
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
|
||||
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
|
||||
If yes, record them as facts with the most appropriate category and confidence.
|
||||
|
||||
{correction_hint}
|
||||
|
||||
Memory Section Guidelines:
|
||||
|
||||
**User Context** (Current state - concise summaries):
|
||||
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
|
||||
Example: Core contributor, project names with metrics (16k+ stars), technical stack
|
||||
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
|
||||
Example: Bilingual capabilities, specific interest areas, expertise domains
|
||||
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
|
||||
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
|
||||
Include: Active implementation work, troubleshooting issues, market/research interests
|
||||
Note: This captures SEVERAL concurrent focus areas, not just one task
|
||||
|
||||
**History** (Temporal context - rich paragraphs; updated incrementally each conversation):
|
||||
- recentMonths: Running log of recent work and activities (4-6 sentences or 1-2 paragraphs)
|
||||
Scope: Accumulate from each conversation — any task the user worked on, problems solved, technologies touched
|
||||
Update policy: Set shouldUpdate=true whenever the conversation contains any task, discussion, or work activity; integrate new content into existing text rather than overwriting
|
||||
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
|
||||
- earlierContext: Important patterns promoted from recentMonths over time (3-5 sentences or 1 paragraph)
|
||||
Scope: Recurring themes and established patterns across multiple conversations
|
||||
Update policy: Set shouldUpdate=true when a pattern repeats or an older item in recentMonths should be archived here
|
||||
Include: Past projects, learning journeys, established patterns
|
||||
- longTermBackground: Persistent foundational context that rarely changes (2-4 sentences)
|
||||
Scope: Core expertise, fundamental working style, long-standing interests
|
||||
Update policy: Set shouldUpdate=true only when a genuinely new foundational fact emerges
|
||||
Include: Core expertise, longstanding interests, fundamental working style
|
||||
|
||||
**Facts Extraction**:
|
||||
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
|
||||
- Include proper nouns (company names, project names, technology names)
|
||||
- Preserve technical terminology and version numbers
|
||||
- Categories:
|
||||
* preference: Tools, styles, approaches user prefers/dislikes
|
||||
* knowledge: Specific expertise, technologies mastered, domain knowledge
|
||||
* context: Background facts (job title, projects, locations, languages)
|
||||
* behavior: Working patterns, communication habits, problem-solving approaches
|
||||
* goal: Stated objectives, learning targets, project ambitions
|
||||
* correction: Explicit agent mistakes or user corrections, including the correct approach
|
||||
- Confidence levels:
|
||||
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
|
||||
* 0.7-0.8: Strongly implied from actions/discussions
|
||||
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
|
||||
|
||||
**What Goes Where**:
|
||||
- workContext: Current job, active projects, primary tech stack
|
||||
- personalContext: Languages, personality, interests outside direct work tasks
|
||||
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
|
||||
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
|
||||
- recentMonths: Cumulative log of recent work — update from EVERY conversation with meaningful content
|
||||
- earlierContext: Recurring patterns promoted from recentMonths; update when patterns solidify
|
||||
- longTermBackground: Unchanging foundational facts about the user; update only for new foundational info
|
||||
|
||||
**Language Requirement**:
|
||||
- ALL summaries and fact content MUST be written in Chinese (Simplified)
|
||||
- Keep technical terms, proper nouns, product names, and company names in their original form (e.g., DeepSeek, LangGraph, GitHub)
|
||||
- Surrounding descriptive text must be in Chinese regardless of the conversation language
|
||||
|
||||
Output Format (JSON):
|
||||
{{
|
||||
"user": {{
|
||||
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"history": {{
|
||||
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"newFacts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
],
|
||||
"factsToRemove": ["fact_id_1", "fact_id_2"]
|
||||
}}
|
||||
|
||||
Important Rules:
|
||||
- For recentMonths: set shouldUpdate=true whenever the conversation contains any task, work, or substantive discussion — this field accumulates like a log
|
||||
- For earlierContext / longTermBackground: set shouldUpdate=true only when a new pattern solidifies or a foundational fact changes
|
||||
- For user context fields: set shouldUpdate=true if there's meaningful new information
|
||||
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
|
||||
- Include specific metrics, version numbers, and proper nouns in facts
|
||||
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
|
||||
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
|
||||
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
|
||||
- Remove facts that are contradicted by new information
|
||||
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
|
||||
Keep 3-5 concurrent focus themes that are still active and relevant
|
||||
- For history sections, integrate new information chronologically into appropriate time period
|
||||
- Preserve technical accuracy - keep exact names of technologies, companies, projects
|
||||
- Focus on information useful for future interactions and personalization
|
||||
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
|
||||
session-specific and ephemeral — they will not be accessible in future sessions.
|
||||
Recording upload events causes confusion in subsequent conversations.
|
||||
- CRITICAL: Write ALL summary text and fact content in Chinese (Simplified). Technical terms and proper nouns (product names, company names, technology names) may remain in their original language, but all other text must be in Chinese.
|
||||
|
||||
Return ONLY valid JSON, no explanation or markdown."""
|
||||
|
||||
|
||||
# Prompt template for extracting facts from a single message
|
||||
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
|
||||
|
||||
Message:
|
||||
{message}
|
||||
|
||||
Extract facts in this JSON format:
|
||||
{{
|
||||
"facts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
]
|
||||
}}
|
||||
|
||||
Categories:
|
||||
- preference: User preferences (likes/dislikes, styles, tools)
|
||||
- knowledge: User's expertise or knowledge areas
|
||||
- context: Background context (location, job, projects)
|
||||
- behavior: Behavioral patterns
|
||||
- goal: User's goals or objectives
|
||||
- correction: Explicit corrections or mistakes to avoid repeating
|
||||
|
||||
Rules:
|
||||
- Only extract clear, specific facts
|
||||
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
|
||||
- Skip vague or temporary information
|
||||
- Write all fact content in Chinese (Simplified); technical terms and proper nouns may remain in their original language
|
||||
|
||||
Return ONLY valid JSON."""
|
||||
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
|
||||
"""Count tokens in text using tiktoken.
|
||||
|
||||
Args:
|
||||
text: The text to count tokens for.
|
||||
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
|
||||
|
||||
Returns:
|
||||
The number of tokens in the text.
|
||||
"""
|
||||
if not TIKTOKEN_AVAILABLE:
|
||||
return len(text) // 4
|
||||
|
||||
try:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
return len(encoding.encode(text))
|
||||
except Exception:
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
|
||||
"""Coerce a confidence-like value to a bounded float in [0, 1].
|
||||
|
||||
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
|
||||
to the default before clamping, preventing them from dominating ranking.
|
||||
The ``default`` parameter is assumed to be a finite value.
|
||||
"""
|
||||
try:
|
||||
confidence = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return max(0.0, min(1.0, default))
|
||||
if not math.isfinite(confidence):
|
||||
return max(0.0, min(1.0, default))
|
||||
return max(0.0, min(1.0, confidence))
|
||||
|
||||
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
|
||||
"""Format memory data for injection into system prompt.
|
||||
|
||||
Args:
|
||||
memory_data: The memory data dictionary.
|
||||
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
|
||||
|
||||
Returns:
|
||||
Formatted memory string for system prompt injection.
|
||||
"""
|
||||
if not memory_data:
|
||||
return ""
|
||||
|
||||
sections = []
|
||||
|
||||
# Format user context
|
||||
user_data = memory_data.get("user", {})
|
||||
if user_data:
|
||||
user_sections = []
|
||||
|
||||
work_ctx = user_data.get("workContext", {})
|
||||
if work_ctx.get("summary"):
|
||||
user_sections.append(f"Work: {work_ctx['summary']}")
|
||||
|
||||
personal_ctx = user_data.get("personalContext", {})
|
||||
if personal_ctx.get("summary"):
|
||||
user_sections.append(f"Personal: {personal_ctx['summary']}")
|
||||
|
||||
top_of_mind = user_data.get("topOfMind", {})
|
||||
if top_of_mind.get("summary"):
|
||||
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
|
||||
|
||||
if user_sections:
|
||||
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
|
||||
|
||||
# Format history
|
||||
history_data = memory_data.get("history", {})
|
||||
if history_data:
|
||||
history_sections = []
|
||||
|
||||
recent = history_data.get("recentMonths", {})
|
||||
if recent.get("summary"):
|
||||
history_sections.append(f"Recent: {recent['summary']}")
|
||||
|
||||
earlier = history_data.get("earlierContext", {})
|
||||
if earlier.get("summary"):
|
||||
history_sections.append(f"Earlier: {earlier['summary']}")
|
||||
|
||||
background = history_data.get("longTermBackground", {})
|
||||
if background.get("summary"):
|
||||
history_sections.append(f"Background: {background['summary']}")
|
||||
|
||||
if history_sections:
|
||||
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
|
||||
|
||||
# Format facts (sorted by confidence; include as many as token budget allows)
|
||||
facts_data = memory_data.get("facts", [])
|
||||
if isinstance(facts_data, list) and facts_data:
|
||||
ranked_facts = sorted(
|
||||
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
|
||||
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
base_text = "\n\n".join(sections)
|
||||
base_tokens = _count_tokens(base_text) if base_text else 0
|
||||
facts_header = "Facts:\n"
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
|
||||
running_tokens = base_tokens + separator_tokens
|
||||
|
||||
fact_lines: list[str] = []
|
||||
for fact in ranked_facts:
|
||||
content_value = fact.get("content")
|
||||
if not isinstance(content_value, str):
|
||||
continue
|
||||
content = content_value.strip()
|
||||
if not content:
|
||||
continue
|
||||
category = str(fact.get("category", "context")).strip() or "context"
|
||||
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
|
||||
source_error = fact.get("sourceError")
|
||||
if category == "correction" and isinstance(source_error, str) and source_error.strip():
|
||||
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
|
||||
else:
|
||||
line = f"- [{category} | {confidence:.2f}] {content}"
|
||||
|
||||
line_text = ("\n" + line) if fact_lines else line
|
||||
line_tokens = _count_tokens(line_text)
|
||||
|
||||
if running_tokens + line_tokens <= max_tokens:
|
||||
fact_lines.append(line)
|
||||
running_tokens += line_tokens
|
||||
else:
|
||||
break
|
||||
|
||||
if fact_lines:
|
||||
sections.append("Facts:\n" + "\n".join(fact_lines))
|
||||
|
||||
if not sections:
|
||||
return ""
|
||||
|
||||
result = "\n\n".join(sections)
|
||||
|
||||
token_count = _count_tokens(result)
|
||||
if token_count > max_tokens:
|
||||
char_per_token = len(result) / token_count
|
||||
target_chars = int(max_tokens * char_per_token * 0.95)
|
||||
result = result[:target_chars] + "\n..."
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_conversation_for_update(messages: list[Any]) -> str:
|
||||
"""Format conversation messages for memory update prompt.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
|
||||
Returns:
|
||||
Formatted conversation string.
|
||||
"""
|
||||
lines = []
|
||||
for msg in messages:
|
||||
role = getattr(msg, "type", "unknown")
|
||||
content = getattr(msg, "content", str(msg))
|
||||
|
||||
# Handle content that might be a list (multimodal)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for p in content:
|
||||
if isinstance(p, str):
|
||||
text_parts.append(p)
|
||||
elif isinstance(p, dict):
|
||||
text_val = p.get("text")
|
||||
if isinstance(text_val, str):
|
||||
text_parts.append(text_val)
|
||||
content = " ".join(text_parts) if text_parts else str(content)
|
||||
|
||||
# Strip uploaded_files tags from human messages to avoid persisting
|
||||
# ephemeral file path info into long-term memory. Skip the turn entirely
|
||||
# when nothing remains after stripping (upload-only message).
|
||||
if role == "human":
|
||||
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Truncate very long messages
|
||||
if len(str(content)) > 1000:
|
||||
content = str(content)[:1000] + "..."
|
||||
|
||||
if role == "human":
|
||||
lines.append(f"User: {content}")
|
||||
elif role == "ai":
|
||||
lines.append(f"Assistant: {content}")
|
||||
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MEMORY_UPDATE_PROMPT",
|
||||
"FACT_EXTRACTION_PROMPT",
|
||||
"format_memory_for_injection",
|
||||
"format_conversation_for_update",
|
||||
"_count_tokens",
|
||||
"_coerce_confidence",
|
||||
]
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Memory V1 — 防抖异步更新队列(原版 DeerFlow MemoryUpdateQueue)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
def _memory_enabled() -> bool:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().enabled
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationContext:
|
||||
"""Context for a conversation to be processed for memory update."""
|
||||
thread_id: str
|
||||
messages: list[Any]
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
agent_name: str | None = None
|
||||
user_id: str | None = None
|
||||
correction_detected: bool = False
|
||||
reinforcement_detected: bool = False
|
||||
|
||||
|
||||
class MemoryUpdateQueue:
|
||||
"""Queue for memory updates with debounce mechanism.
|
||||
|
||||
Collects conversation contexts and processes them after a configurable
|
||||
debounce period. Multiple conversations received within the debounce
|
||||
window are deduplicated by (thread_id, user_id, agent_name).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue: list[ConversationContext] = []
|
||||
self._lock = threading.Lock()
|
||||
self._timer: threading.Timer | None = None
|
||||
self._processing = False
|
||||
|
||||
@staticmethod
|
||||
def _queue_key(thread_id: str, user_id: str | None, agent_name: str | None) -> tuple[str, str | None, str | None]:
|
||||
return (thread_id, user_id, agent_name)
|
||||
|
||||
def add(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation to the update queue (with debounce)."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._reset_timer()
|
||||
|
||||
logger.info("V1 memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def add_nowait(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation and start processing immediately in the background."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._schedule_timer(0)
|
||||
|
||||
logger.info("V1 memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def _enqueue_locked(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
user_id: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
) -> None:
|
||||
queue_key = self._queue_key(thread_id, user_id, agent_name)
|
||||
|
||||
existing = next(
|
||||
(ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) == queue_key),
|
||||
None,
|
||||
)
|
||||
|
||||
merged_correction = correction_detected or (existing.correction_detected if existing is not None else False)
|
||||
merged_reinforcement = reinforcement_detected or (existing.reinforcement_detected if existing is not None else False)
|
||||
|
||||
context = ConversationContext(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=merged_correction,
|
||||
reinforcement_detected=merged_reinforcement,
|
||||
)
|
||||
|
||||
self._queue = [ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) != queue_key]
|
||||
self._queue.append(context)
|
||||
|
||||
def _reset_timer(self) -> None:
|
||||
cfg = _get_v1_cfg()
|
||||
self._schedule_timer(cfg.debounce_seconds)
|
||||
logger.debug("V1 memory update timer set for %ss", cfg.debounce_seconds)
|
||||
|
||||
def _schedule_timer(self, delay_seconds: float) -> None:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = threading.Timer(delay_seconds, self._process_queue)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _process_queue(self) -> None:
|
||||
from deerflow.agents.memory.v1.updater import MemoryUpdater
|
||||
|
||||
with self._lock:
|
||||
if self._processing:
|
||||
self._schedule_timer(0)
|
||||
return
|
||||
if not self._queue:
|
||||
return
|
||||
self._processing = True
|
||||
contexts_to_process = self._queue.copy()
|
||||
self._queue.clear()
|
||||
self._timer = None
|
||||
|
||||
logger.info("V1 memory: processing %d queued updates", len(contexts_to_process))
|
||||
|
||||
try:
|
||||
updater = MemoryUpdater()
|
||||
for context in contexts_to_process:
|
||||
try:
|
||||
logger.info("V1 memory: updating for thread %s", context.thread_id)
|
||||
success = updater.update_memory(
|
||||
messages=context.messages,
|
||||
thread_id=context.thread_id,
|
||||
agent_name=context.agent_name,
|
||||
correction_detected=context.correction_detected,
|
||||
reinforcement_detected=context.reinforcement_detected,
|
||||
user_id=context.user_id,
|
||||
)
|
||||
if success:
|
||||
logger.info("V1 memory: updated successfully for thread %s", context.thread_id)
|
||||
else:
|
||||
logger.warning("V1 memory: update skipped/failed for thread %s", context.thread_id)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("V1 memory: error updating for thread %s: %s", context.thread_id, e)
|
||||
|
||||
if len(contexts_to_process) > 1:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processing = False
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force immediate processing of the queue."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._process_queue()
|
||||
|
||||
def flush_nowait(self) -> None:
|
||||
"""Start queue processing immediately in a background thread."""
|
||||
with self._lock:
|
||||
self._schedule_timer(0)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear the queue without processing (for testing)."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._queue.clear()
|
||||
self._processing = False
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._queue)
|
||||
|
||||
@property
|
||||
def is_processing(self) -> bool:
|
||||
with self._lock:
|
||||
return self._processing
|
||||
|
||||
|
||||
_memory_queue: MemoryUpdateQueue | None = None
|
||||
_queue_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_queue() -> MemoryUpdateQueue:
|
||||
"""Get the global memory update queue singleton."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is None:
|
||||
_memory_queue = MemoryUpdateQueue()
|
||||
return _memory_queue
|
||||
|
||||
|
||||
def reset_memory_queue() -> None:
|
||||
"""Reset the global memory queue (for testing)."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is not None:
|
||||
_memory_queue.clear()
|
||||
_memory_queue = None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConversationContext",
|
||||
"MemoryUpdateQueue",
|
||||
"get_memory_queue",
|
||||
"reset_memory_queue",
|
||||
]
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
"""Memory V1 — 存储层(原版 DeerFlow 格式)。
|
||||
|
||||
JSON 结构:
|
||||
{
|
||||
"version": "1.0",
|
||||
"lastUpdated": "...",
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground":{"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"facts": []
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def utc_now_iso_z() -> str:
|
||||
"""Current UTC time as ISO-8601 with ``Z`` suffix."""
|
||||
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
|
||||
|
||||
|
||||
def create_empty_memory() -> dict[str, Any]:
|
||||
"""Create an empty memory structure (new format)."""
|
||||
return {
|
||||
"version": "1.0",
|
||||
"lastUpdated": utc_now_iso_z(),
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"facts": [],
|
||||
}
|
||||
|
||||
|
||||
def _migrate_legacy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把老格式 {userContext: {workContext: str}} 迁移到新格式 {user: {workContext: {summary, updatedAt}}}。"""
|
||||
if "userContext" not in data:
|
||||
return data
|
||||
|
||||
now = utc_now_iso_z()
|
||||
old_ctx = data.pop("userContext", {})
|
||||
data["user"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in old_ctx.items()
|
||||
}
|
||||
|
||||
# history 字段可能是纯字符串,需要包裹
|
||||
if "history" in data:
|
||||
hist = data["history"]
|
||||
data["history"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in hist.items()
|
||||
}
|
||||
|
||||
# 确保 user 包含所有必要字段
|
||||
base = create_empty_memory()
|
||||
for key in ("user", "history"):
|
||||
section = base[key]
|
||||
for field in section:
|
||||
if field not in data.get(key, {}):
|
||||
data.setdefault(key, {})[field] = section[field]
|
||||
|
||||
data["version"] = "1.0"
|
||||
data.pop("updatedAt", None)
|
||||
return data
|
||||
|
||||
|
||||
class FileMemoryStorage:
|
||||
"""File-based memory storage provider (原版 DeerFlow 实现)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Per-(user_id, agent_name) cache: value = (memory_data, file_mtime)
|
||||
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
def _validate_agent_name(self, agent_name: str) -> None:
|
||||
if not agent_name:
|
||||
raise ValueError("Agent name must be a non-empty string.")
|
||||
if not AGENT_NAME_PATTERN.match(agent_name):
|
||||
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
|
||||
|
||||
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
|
||||
paths = get_paths()
|
||||
if user_id is not None:
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.user_agent_memory_file(user_id, agent_name)
|
||||
return paths.user_memory_file(user_id)
|
||||
# Legacy: no user_id
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.agent_memory_file(agent_name)
|
||||
return paths.memory_file
|
||||
|
||||
def _load_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
if not file_path.exists():
|
||||
return create_empty_memory()
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return create_empty_memory()
|
||||
data = _migrate_legacy(data)
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning("Failed to load memory file %s: %s", file_path, e)
|
||||
return create_empty_memory()
|
||||
|
||||
@staticmethod
|
||||
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
|
||||
return (user_id, agent_name)
|
||||
|
||||
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data (cached with file modification time check)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
current_mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
cached = self._memory_cache.get(cache_key)
|
||||
if cached is not None and cached[1] == current_mtime:
|
||||
return cached[0]
|
||||
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, current_mtime)
|
||||
|
||||
return memory_data
|
||||
|
||||
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Reload memory data from file, forcing cache invalidation."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
return memory_data
|
||||
|
||||
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Save memory data to file and update cache (atomic write)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
|
||||
|
||||
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(memory_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
temp_path.replace(file_path)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
logger.info("Memory saved to %s", file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error("Failed to save memory file: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
_storage_instance: FileMemoryStorage | None = None
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_storage() -> FileMemoryStorage:
|
||||
"""Get the global FileMemoryStorage singleton."""
|
||||
global _storage_instance
|
||||
if _storage_instance is not None:
|
||||
return _storage_instance
|
||||
with _storage_lock:
|
||||
if _storage_instance is None:
|
||||
_storage_instance = FileMemoryStorage()
|
||||
return _storage_instance
|
||||
|
||||
|
||||
__all__ = [
|
||||
"create_empty_memory",
|
||||
"utc_now_iso_z",
|
||||
"FileMemoryStorage",
|
||||
"get_memory_storage",
|
||||
]
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""Hooks fired before summarization removes messages from state(原版 DeerFlow)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
|
||||
|
||||
def memory_flush_hook(event: object) -> None:
|
||||
"""Flush messages about to be summarized into the V1 memory queue.
|
||||
|
||||
``event`` is expected to be a ``SummarizationEvent``-like object with:
|
||||
- ``thread_id: str``
|
||||
- ``messages_to_summarize: list``
|
||||
- ``agent_name: str | None``
|
||||
- ``runtime: RunnableConfig``
|
||||
"""
|
||||
if not get_memory_config().enabled:
|
||||
return
|
||||
|
||||
thread_id = getattr(event, "thread_id", None)
|
||||
if not thread_id:
|
||||
return
|
||||
|
||||
messages_to_summarize = getattr(event, "messages_to_summarize", [])
|
||||
filtered_messages = filter_messages_for_memory(list(messages_to_summarize))
|
||||
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
|
||||
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
|
||||
if not user_messages or not assistant_messages:
|
||||
return
|
||||
|
||||
correction_detected = detect_correction(filtered_messages)
|
||||
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
|
||||
|
||||
try:
|
||||
from deerflow.runtime.user_context import resolve_runtime_user_id
|
||||
runtime = getattr(event, "runtime", None)
|
||||
user_id = resolve_runtime_user_id(runtime) if runtime is not None else None
|
||||
except Exception: # noqa: BLE001
|
||||
user_id = None
|
||||
|
||||
agent_name = getattr(event, "agent_name", None)
|
||||
queue = get_memory_queue()
|
||||
queue.add_nowait(
|
||||
thread_id=thread_id,
|
||||
messages=filtered_messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["memory_flush_hook"]
|
||||
+495
@@ -0,0 +1,495 @@
|
||||
"""Memory V1 — updater(原版 DeerFlow MemoryUpdater)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.v1.prompt import (
|
||||
MEMORY_UPDATE_PROMPT,
|
||||
format_conversation_for_update,
|
||||
)
|
||||
from deerflow.agents.memory.v1.storage import (
|
||||
create_empty_memory,
|
||||
get_memory_storage,
|
||||
utc_now_iso_z,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=4,
|
||||
thread_name_prefix="v1-memory-updater-sync",
|
||||
)
|
||||
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
"""返回 V1MemoryConfig(含 max_facts, fact_confidence_threshold, model_name)。"""
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 向后兼容的便捷函数(供 gateway router 等调用)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().reload(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
storage = get_memory_storage()
|
||||
if not storage.save(memory_data, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save imported memory data")
|
||||
return storage.load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
cleared = create_empty_memory()
|
||||
if not get_memory_storage().save(cleared, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save cleared memory data")
|
||||
return cleared
|
||||
|
||||
|
||||
def _validate_confidence(confidence: float) -> float:
|
||||
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
|
||||
raise ValueError("confidence")
|
||||
return confidence
|
||||
|
||||
|
||||
def create_memory_fact(
|
||||
content: str,
|
||||
category: str = "context",
|
||||
confidence: float = 0.5,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_content = content.strip()
|
||||
if not normalized_content:
|
||||
raise ValueError("content")
|
||||
normalized_category = category.strip() or "context"
|
||||
validated_confidence = _validate_confidence(confidence)
|
||||
now = utc_now_iso_z()
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
facts = list(memory_data.get("facts", []))
|
||||
facts.append({
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": normalized_category,
|
||||
"confidence": validated_confidence,
|
||||
"createdAt": now,
|
||||
"source": "manual",
|
||||
})
|
||||
updated_memory["facts"] = facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save memory data after creating fact")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
facts = memory_data.get("facts", [])
|
||||
updated_facts = [f for f in facts if f.get("id") != fact_id]
|
||||
if len(updated_facts) == len(facts):
|
||||
raise KeyError(fact_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def update_memory_fact(
|
||||
fact_id: str,
|
||||
content: str | None = None,
|
||||
category: str | None = None,
|
||||
confidence: float | None = None,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_facts: list[dict[str, Any]] = []
|
||||
found = False
|
||||
for fact in memory_data.get("facts", []):
|
||||
if fact.get("id") == fact_id:
|
||||
found = True
|
||||
updated_fact = dict(fact)
|
||||
if content is not None:
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
raise ValueError("content")
|
||||
updated_fact["content"] = normalized
|
||||
if category is not None:
|
||||
updated_fact["category"] = category.strip() or "context"
|
||||
if confidence is not None:
|
||||
updated_fact["confidence"] = _validate_confidence(confidence)
|
||||
updated_facts.append(updated_fact)
|
||||
else:
|
||||
updated_facts.append(fact)
|
||||
if not found:
|
||||
raise KeyError(fact_id)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内容提取辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
"""从 LLM response content(str 或 list of blocks)提取纯文本。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
pieces: list[str] = []
|
||||
pending: list[str] = []
|
||||
|
||||
def flush():
|
||||
if pending:
|
||||
pieces.append("".join(pending))
|
||||
pending.clear()
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
pending.append(block)
|
||||
elif isinstance(block, dict):
|
||||
flush()
|
||||
text_val = block.get("text")
|
||||
if isinstance(text_val, str):
|
||||
pieces.append(text_val)
|
||||
flush()
|
||||
return "\n".join(pieces)
|
||||
return str(content)
|
||||
|
||||
|
||||
_UPLOAD_SENTENCE_RE = re.compile(
|
||||
r"[^.!?]*\b(?:"
|
||||
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
|
||||
r"|file\s+upload"
|
||||
r"|/mnt/user-data/uploads/"
|
||||
r"|<uploaded_files>"
|
||||
r")[^.!?]*[.!?]?\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""移除记忆中关于文件上传事件的句子(上传文件是会话级临时的)。"""
|
||||
for section in ("user", "history"):
|
||||
section_data = memory_data.get(section, {})
|
||||
for _key, val in section_data.items():
|
||||
if isinstance(val, dict) and "summary" in val:
|
||||
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
|
||||
cleaned = re.sub(r" +", " ", cleaned)
|
||||
val["summary"] = cleaned
|
||||
|
||||
facts = memory_data.get("facts", [])
|
||||
if facts:
|
||||
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
|
||||
return memory_data
|
||||
|
||||
|
||||
def _fact_content_key(content: Any) -> str | None:
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
stripped = content.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
return stripped.casefold()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryUpdater(原版核心类)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class MemoryUpdater:
|
||||
"""Updates memory using LLM based on conversation context."""
|
||||
|
||||
def __init__(self, model_name: str | None = None) -> None:
|
||||
self._model_name = model_name
|
||||
|
||||
def _get_model(self):
|
||||
from deerflow.models import create_chat_model
|
||||
cfg = _get_v1_cfg()
|
||||
model_name = self._model_name or cfg.model_name
|
||||
return create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
|
||||
def _build_correction_hint(self, correction_detected: bool, reinforcement_detected: bool) -> str:
|
||||
correction_hint = ""
|
||||
if correction_detected:
|
||||
correction_hint = (
|
||||
"IMPORTANT: Explicit correction signals were detected in this conversation. "
|
||||
"Pay special attention to what the agent got wrong, what the user corrected, "
|
||||
"and record the correct approach as a fact with category "
|
||||
'"correction" and confidence >= 0.95 when appropriate.'
|
||||
)
|
||||
if reinforcement_detected:
|
||||
reinforcement_hint = (
|
||||
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
|
||||
"The user explicitly confirmed the agent's approach was correct or helpful. "
|
||||
"Record the confirmed approach, style, or preference as a fact with category "
|
||||
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
|
||||
)
|
||||
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
|
||||
return correction_hint
|
||||
|
||||
def _prepare_update_prompt(
|
||||
self,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
user_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], str] | None:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
config = get_memory_config()
|
||||
if not config.enabled or not messages:
|
||||
return None
|
||||
|
||||
current_memory = get_memory_data(agent_name, user_id=user_id)
|
||||
conversation_text = format_conversation_for_update(messages)
|
||||
if not conversation_text.strip():
|
||||
return None
|
||||
|
||||
correction_hint = self._build_correction_hint(correction_detected, reinforcement_detected)
|
||||
prompt = MEMORY_UPDATE_PROMPT.format(
|
||||
current_memory=json.dumps(current_memory, indent=2, ensure_ascii=False),
|
||||
conversation=conversation_text,
|
||||
correction_hint=correction_hint,
|
||||
)
|
||||
return current_memory, prompt
|
||||
|
||||
def _finalize_update(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
response_content: Any,
|
||||
thread_id: str | None,
|
||||
agent_name: str | None,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
response_text = _extract_text(response_content).strip()
|
||||
if response_text.startswith("```"):
|
||||
lines = response_text.split("\n")
|
||||
response_text = "\n".join(lines[1:-1] if lines[-1] == "```" else lines[1:])
|
||||
|
||||
update_data = json.loads(response_text)
|
||||
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
|
||||
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
|
||||
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
|
||||
|
||||
async def aupdate_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
return await asyncio.to_thread(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _do_update_memory_sync(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
prepared = self._prepare_update_prompt(
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
if prepared is None:
|
||||
return False
|
||||
|
||||
current_memory, prompt = prepared
|
||||
model = self._get_model()
|
||||
response = model.invoke(prompt, config={"run_name": "v1_memory_agent"})
|
||||
return self._finalize_update(
|
||||
current_memory=current_memory,
|
||||
response_content=response.content,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("V1 memory: failed to parse LLM response: %s", e)
|
||||
return False
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("V1 memory update failed: %s", e)
|
||||
return False
|
||||
|
||||
def update_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop is not None and loop.is_running():
|
||||
try:
|
||||
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
return future.result()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("V1 memory: failed to offload update to executor")
|
||||
return False
|
||||
|
||||
return self._do_update_memory_sync(
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _apply_updates(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
update_data: dict[str, Any],
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = _get_v1_cfg()
|
||||
now = utc_now_iso_z()
|
||||
|
||||
# 更新 user context 段
|
||||
user_updates = update_data.get("user", {})
|
||||
for section in ("workContext", "personalContext", "topOfMind"):
|
||||
section_data = user_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("user", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 更新 history 段
|
||||
history_updates = update_data.get("history", {})
|
||||
for section in ("recentMonths", "earlierContext", "longTermBackground"):
|
||||
section_data = history_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("history", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 删除事实
|
||||
facts_to_remove = set(update_data.get("factsToRemove", []))
|
||||
if facts_to_remove:
|
||||
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
|
||||
|
||||
# 新增事实(去重 + 置信度过滤)
|
||||
existing_keys = {
|
||||
key for key in (_fact_content_key(f.get("content")) for f in current_memory.get("facts", []))
|
||||
if key is not None
|
||||
}
|
||||
|
||||
for fact in update_data.get("newFacts", []):
|
||||
confidence = fact.get("confidence", 0.5)
|
||||
if confidence < cfg.fact_confidence_threshold:
|
||||
continue
|
||||
raw_content = fact.get("content", "")
|
||||
if not isinstance(raw_content, str):
|
||||
continue
|
||||
normalized_content = raw_content.strip()
|
||||
fact_key = _fact_content_key(normalized_content)
|
||||
if fact_key is not None and fact_key in existing_keys:
|
||||
continue
|
||||
|
||||
fact_entry: dict[str, Any] = {
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": fact.get("category", "context"),
|
||||
"confidence": confidence,
|
||||
"createdAt": now,
|
||||
"source": thread_id or "unknown",
|
||||
}
|
||||
source_error = fact.get("sourceError")
|
||||
if isinstance(source_error, str) and source_error.strip():
|
||||
fact_entry["sourceError"] = source_error.strip()
|
||||
|
||||
current_memory["facts"].append(fact_entry)
|
||||
if fact_key is not None:
|
||||
existing_keys.add(fact_key)
|
||||
|
||||
# 超出上限时驱逐置信度最低的事实
|
||||
if len(current_memory["facts"]) > cfg.max_facts:
|
||||
current_memory["facts"] = sorted(
|
||||
current_memory["facts"],
|
||||
key=lambda f: f.get("confidence", 0),
|
||||
reverse=True,
|
||||
)[:cfg.max_facts]
|
||||
|
||||
return current_memory
|
||||
|
||||
|
||||
def update_memory_from_conversation(
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
updater = MemoryUpdater()
|
||||
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryUpdater",
|
||||
"update_memory_from_conversation",
|
||||
"get_memory_data",
|
||||
"reload_memory_data",
|
||||
"import_memory_data",
|
||||
"clear_memory_data",
|
||||
"create_memory_fact",
|
||||
"delete_memory_fact",
|
||||
"update_memory_fact",
|
||||
]
|
||||
+7
-7
@@ -171,21 +171,21 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
|
||||
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
|
||||
seconds = max(1, round(wait_ms / 1000))
|
||||
reason_text = "provider is busy" if reason == "busy" else "provider request failed temporarily"
|
||||
return f"LLM request retry {attempt}/{self.retry_max_attempts}: {reason_text}. Retrying in {seconds}s."
|
||||
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
|
||||
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
|
||||
|
||||
def _build_circuit_breaker_message(self) -> str:
|
||||
return "The configured LLM provider is currently unavailable due to continuous failures. Circuit breaker is engaged to protect the system. Please wait a moment before trying again."
|
||||
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
|
||||
|
||||
def _build_user_message(self, exc: BaseException, reason: str) -> str:
|
||||
detail = _extract_error_detail(exc)
|
||||
if reason == "quota":
|
||||
return "The configured LLM provider rejected the request because the account is out of quota, billing is unavailable, or usage is restricted. Please fix the provider account and try again."
|
||||
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
|
||||
if reason == "auth":
|
||||
return "The configured LLM provider rejected the request because authentication or access is invalid. Please check the provider credentials and try again."
|
||||
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
|
||||
if reason in {"busy", "transient"}:
|
||||
return "The configured LLM provider is temporarily unavailable after multiple retries. Please wait a moment and continue the conversation."
|
||||
return f"LLM request failed: {detail}"
|
||||
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
|
||||
return f"大模型因「{detail}」原因调用失败。"
|
||||
|
||||
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
|
||||
try:
|
||||
|
||||
+11
-4
@@ -90,16 +90,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
|
||||
state_schema = MemoryMiddlewareState
|
||||
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None, injection_enabled: bool = True):
|
||||
"""初始化 MemoryMiddleware。
|
||||
|
||||
Args:
|
||||
agent_name: 提供时记忆按 agent 隔离;None 用默认 agent。
|
||||
memory_config: 显式记忆配置;省略时用全局配置。
|
||||
injection_enabled: 运行时开关;``False`` 时跳过 Hindsight 动态召回注入。
|
||||
"""
|
||||
super().__init__()
|
||||
self._agent_name = agent_name
|
||||
self._memory_config = memory_config
|
||||
self._injection_enabled = injection_enabled
|
||||
|
||||
# ---- 运行时上下文解析 -------------------------------------------------
|
||||
|
||||
@@ -115,15 +117,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
def _resolve_agent_id(self, runtime: Runtime | None) -> str:
|
||||
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。"""
|
||||
def _resolve_agent_id(self, runtime: Runtime | None) -> str | None:
|
||||
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。
|
||||
|
||||
无 agent 上下文时返回 None,使 V1 记忆存入全局文件(users/{id}/memory.json)。
|
||||
"""
|
||||
if self._agent_name:
|
||||
return self._agent_name
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
agent_id = runtime.context.get("agent_id")
|
||||
if agent_id:
|
||||
return str(agent_id)
|
||||
return "default"
|
||||
return None
|
||||
|
||||
# ---- wrap_model_call:Hindsight 动态召回注入 -------------------------
|
||||
|
||||
@@ -133,6 +138,8 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
|
||||
背景信息而非用户的新输入。
|
||||
"""
|
||||
if not self._injection_enabled:
|
||||
return None
|
||||
config = self._memory_config or get_memory_config()
|
||||
if not config.enabled:
|
||||
return None
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Middleware for intercepting skill calls and stopping execution when specific skills are invoked."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command, interrupt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_name_task(text: str) -> tuple[str, str]:
|
||||
"""Extract `agent_name` and `task_content` from a bash invocation that triggers
|
||||
a sub-agent via ``python3 scripts/agent_orchestration.py "<agent>" "<task>"``.
|
||||
|
||||
Returns empty strings when no match is found so the caller can treat the
|
||||
absence of a match as "not a target skill call".
|
||||
"""
|
||||
try:
|
||||
pattern = r'\.py\s+"([^"]*)"\s+"([^"]*)"'
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
return match.group(1), match.group(2)
|
||||
return "", ""
|
||||
except Exception:
|
||||
return "", ""
|
||||
|
||||
|
||||
class SkillStopMiddlewareState(AgentState):
|
||||
"""Compatible with the `ThreadState` schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SkillStopMiddleware(AgentMiddleware[SkillStopMiddlewareState]):
|
||||
"""Intercepts skill file reads and stops execution when specific skills are invoked.
|
||||
|
||||
When the model reads a file from a skill directory that matches one of the
|
||||
configured skill names, this middleware:
|
||||
1. Intercepts the tool call before execution
|
||||
2. Returns a Command that stops execution (goes to END)
|
||||
3. Optionally returns a message indicating which skill was triggered
|
||||
|
||||
This is useful for implementing "skill triggers" that pause the agent
|
||||
when certain skills are about to be used.
|
||||
"""
|
||||
|
||||
state_schema = SkillStopMiddlewareState
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
skill_names: list[str],
|
||||
skills_root: str = "/mnt/skills",
|
||||
stop_message: str | None = None,
|
||||
stop_on_match: bool = True,
|
||||
):
|
||||
"""Initialize the skill stop middleware.
|
||||
|
||||
Args:
|
||||
skill_names: List of skill names to intercept (e.g., ["newsletter-generation", "podcast-generation"]).
|
||||
skills_root: Root directory where skills are located (default: "/mnt/skills").
|
||||
stop_message: Optional message to return when stopping. If None, a default message is used.
|
||||
stop_on_match: If True, stop when skill matches. If False, continue execution (for logging only).
|
||||
"""
|
||||
super().__init__()
|
||||
self._skill_names = set(skill_names)
|
||||
self._skills_root = skills_root.rstrip("/")
|
||||
self._stop_message = stop_message
|
||||
self._stop_on_match = stop_on_match
|
||||
|
||||
def _is_target_skill_call(self, tool_call: dict[str, Any]) -> str | None:
|
||||
"""Return the matched skill name if this tool call invokes one of the
|
||||
configured target skills, otherwise ``None``.
|
||||
|
||||
Two invocation patterns are recognized:
|
||||
|
||||
1. **Structured tool call** — the model emits a tool call whose ``name``
|
||||
literally matches one of the configured skill names (e.g. the
|
||||
multi-agent leader calling
|
||||
``agent_orchestration({"agent_name": "...", "task": "..."})``).
|
||||
This is the natural form most LLMs produce when told to "use a
|
||||
skill", and it's the form that drives the roundtable dispatch.
|
||||
|
||||
2. **Bash + argv** — the legacy form where the lead agent reaches into
|
||||
a skill by running ``python3 scripts/<skill-name>.py "<arg1>" "<arg2>"``.
|
||||
Kept for backward compatibility with skills that use bash for
|
||||
dispatch.
|
||||
"""
|
||||
name = tool_call.get("name") or ""
|
||||
lowered_targets = {s.lower() for s in self._skill_names}
|
||||
|
||||
# Pattern 1: structured tool call whose tool name IS the skill name.
|
||||
if name and (name in self._skill_names or name.lower() in lowered_targets):
|
||||
logger.info("SkillStopMiddleware: matched structured skill call name=%s", name)
|
||||
return name
|
||||
|
||||
# Pattern 2: bash invocation with argv-style skill name.
|
||||
if name == "bash" and "args" in tool_call:
|
||||
skill_name, _ = extract_name_task(str(tool_call["args"]))
|
||||
if skill_name and (skill_name in self._skill_names or skill_name.lower() in lowered_targets):
|
||||
logger.info("SkillStopMiddleware: matched bash skill call name=%s", skill_name)
|
||||
return skill_name
|
||||
|
||||
return None
|
||||
|
||||
def _handle_skill_stop(self, request: ToolCallRequest, skill_name: str) -> Command:
|
||||
"""Build the stop response — an interrupt that includes a ToolMessage
|
||||
describing which skill was triggered."""
|
||||
logger.info("Skill stop triggered: skill_name=%s", skill_name)
|
||||
|
||||
tool_call_id = request.tool_call.get("id", "")
|
||||
message_content = self._stop_message or f"⏹️ Skill '{skill_name}' triggered. Execution stopped."
|
||||
|
||||
tool_message = ToolMessage(
|
||||
id=f"skill_stop:{tool_call_id}",
|
||||
content=message_content,
|
||||
tool_call_id=tool_call_id,
|
||||
name="skill_stop",
|
||||
)
|
||||
|
||||
return interrupt(
|
||||
{
|
||||
"messages": [tool_message],
|
||||
"skill_name": skill_name,
|
||||
}
|
||||
)
|
||||
|
||||
@override
|
||||
def wrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
matched_skill = self._is_target_skill_call(request.tool_call)
|
||||
if matched_skill:
|
||||
if self._stop_on_match:
|
||||
return self._handle_skill_stop(request, matched_skill)
|
||||
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
matched_skill = self._is_target_skill_call(request.tool_call)
|
||||
if matched_skill:
|
||||
if self._stop_on_match:
|
||||
return self._handle_skill_stop(request, matched_skill)
|
||||
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
|
||||
return await handler(request)
|
||||
+41
-2
@@ -105,8 +105,16 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
return prompt, user_msg
|
||||
|
||||
def _strip_think_tags(self, text: str) -> str:
|
||||
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, DeepSeek-R1)."""
|
||||
return re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE).strip()
|
||||
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, GLM, DeepSeek-R1).
|
||||
|
||||
Also handles malformed cases: an unclosed `<think>` (response truncated mid-reasoning)
|
||||
or a stray `</think>` (open tag eaten by upstream parsing) — both can otherwise leak
|
||||
raw reasoning into the title.
|
||||
"""
|
||||
text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"<think>[\s\S]*$", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"^[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
|
||||
return text.strip()
|
||||
|
||||
def _parse_title(self, content: object) -> str:
|
||||
"""Normalize model output into a clean title string."""
|
||||
@@ -146,6 +154,36 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
_, user_msg = self._build_title_prompt(state)
|
||||
return {"title": self._fallback_title(user_msg)}
|
||||
|
||||
def _force_disable_thinking(self, model: object) -> None:
|
||||
"""Force-disable thinking on the title model via `extra_body` overrides.
|
||||
|
||||
Title generation never needs reasoning. The model factory only emits
|
||||
disable params when the model config declares `when_thinking_disabled`
|
||||
or a matching `when_thinking_enabled` shape — many user-added reasoning
|
||||
models (e.g. GLM-4.5/5) only set `supports_thinking: true` and would
|
||||
otherwise keep thinking on, which is slow and leaks <think> tags into
|
||||
the title prompt response. Injecting both common OpenAI-compatible
|
||||
disable shapes covers Zhipu GLM (`thinking.type`) and Qwen/vLLM
|
||||
(`chat_template_kwargs.enable_thinking`); other providers ignore
|
||||
unknown body fields.
|
||||
"""
|
||||
if not hasattr(model, "extra_body"):
|
||||
return
|
||||
current = getattr(model, "extra_body", None) or {}
|
||||
merged_chat_template_kwargs = {
|
||||
**(current.get("chat_template_kwargs") or {}),
|
||||
"enable_thinking": False,
|
||||
}
|
||||
merged = {
|
||||
**current,
|
||||
"thinking": {**(current.get("thinking") or {}), "type": "disabled"},
|
||||
"chat_template_kwargs": merged_chat_template_kwargs,
|
||||
}
|
||||
try:
|
||||
model.extra_body = merged
|
||||
except Exception:
|
||||
logger.debug("Could not override extra_body to disable thinking on title model", exc_info=True)
|
||||
|
||||
async def _agenerate_title_result(self, state: TitleMiddlewareState) -> dict | None:
|
||||
"""Generate a title asynchronously and fall back locally on failure."""
|
||||
if not self._should_generate_title(state):
|
||||
@@ -162,6 +200,7 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
model = create_chat_model(name=config.model_name, **model_kwargs)
|
||||
else:
|
||||
model = create_chat_model(**model_kwargs)
|
||||
self._force_disable_thinking(model)
|
||||
response = await model.ainvoke(prompt, config=self._get_runnable_config())
|
||||
title = self._parse_title(response.content)
|
||||
if title:
|
||||
|
||||
@@ -57,11 +57,18 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
|
||||
agent_id: The agent storage/runtime id.
|
||||
|
||||
Returns:
|
||||
AgentConfig instance.
|
||||
AgentConfig instance, or ``None`` when the on-disk artifacts are missing.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the agent directory or config.yaml does not exist.
|
||||
ValueError: If config.yaml cannot be parsed.
|
||||
ValueError: If config.yaml exists but cannot be parsed.
|
||||
|
||||
Tolerance:
|
||||
A missing agent directory or missing config.yaml is **not** an error —
|
||||
the runtime simply falls back to default Lead Agent behavior for that
|
||||
id. This keeps the chat session usable when the DB still has an agent
|
||||
row but its files have been deleted (e.g. data dir moved between
|
||||
machines, or filesystem wiped). The router layer is expected to
|
||||
lazily restore the directory next time the agent is fetched via API.
|
||||
"""
|
||||
|
||||
if agent_id is None:
|
||||
@@ -72,10 +79,20 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
|
||||
config_file = agent_dir / "config.yaml"
|
||||
|
||||
if not agent_dir.exists():
|
||||
raise FileNotFoundError(f"Agent directory not found: {agent_dir}")
|
||||
logger.warning(
|
||||
"Agent directory missing for id '%s' (%s); falling back to default config",
|
||||
agent_id,
|
||||
agent_dir,
|
||||
)
|
||||
return None
|
||||
|
||||
if not config_file.exists() and not (agent_dir / SOUL_FILENAME).exists():
|
||||
raise FileNotFoundError(f"Agent config not found: {config_file}")
|
||||
logger.warning(
|
||||
"Agent has no config.yaml or %s at %s; falling back to default config",
|
||||
SOUL_FILENAME,
|
||||
agent_dir,
|
||||
)
|
||||
return None
|
||||
|
||||
data: dict[str, Any] = {}
|
||||
if config_file.exists():
|
||||
|
||||
@@ -22,6 +22,7 @@ from deerflow.config.sandbox_config import SandboxConfig
|
||||
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
|
||||
from deerflow.config.skill_evolution_runtime import get_overrides_mtime as _get_skill_evolution_overrides_mtime
|
||||
from deerflow.config.skill_evolution_runtime import load_overrides as _load_skill_evolution_overrides
|
||||
from deerflow.config.skill_stop_config import SkillStopConfig
|
||||
from deerflow.config.skills_config import SkillsConfig
|
||||
from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict
|
||||
from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict
|
||||
@@ -117,6 +118,8 @@ class AppConfig(BaseModel):
|
||||
subagents: SubagentsAppConfig = Field(default_factory=SubagentsAppConfig, description="Subagent runtime configuration")
|
||||
guardrails: GuardrailsConfig = Field(default_factory=GuardrailsConfig, description="Guardrail middleware configuration")
|
||||
circuit_breaker: CircuitBreakerConfig = Field(default_factory=CircuitBreakerConfig, description="LLM circuit breaker configuration")
|
||||
excluded_tools: list[str] = Field(default_factory=list, description="Tool names to exclude from the main agent (e.g. ['task', 'ask_clarification'])")
|
||||
skill_stop: SkillStopConfig = Field(default_factory=SkillStopConfig, description="Skill stop middleware configuration")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
database: DatabaseConfig = Field(default_factory=DatabaseConfig, description="Unified database backend configuration")
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
|
||||
+101
-93
@@ -12,21 +12,21 @@
|
||||
injection: {...} # 系统提示词注入
|
||||
security: {...} # 内容安全扫描
|
||||
|
||||
三层配置覆盖
|
||||
两层配置覆盖
|
||||
============
|
||||
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
|
||||
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
|
||||
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
|
||||
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
|
||||
4. Pydantic 默认值
|
||||
1. per-user 覆盖 数据库 ``user_memory_configs`` 表 (经进程内 cache 读取)
|
||||
(有 per-user 记录时自动强制 enabled=True)
|
||||
2. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
|
||||
3. Pydantic 默认值
|
||||
|
||||
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
第 1 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
cache 由 Gateway lifespan 启动时预热,写入时同步更新,保持同步读取路径。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -73,7 +73,7 @@ class HindsightMemoryConfig(BaseModel):
|
||||
description="agent 私有工作 bank 命名模板",
|
||||
)
|
||||
memory_mode: str = Field(
|
||||
default="context",
|
||||
default="hybrid",
|
||||
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
|
||||
)
|
||||
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
|
||||
@@ -113,15 +113,30 @@ class SecurityConfig(BaseModel):
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class V1MemoryConfig(BaseModel):
|
||||
"""原版 DeerFlow V1 记忆 provider 配置 (memory.json + LLM 自动提取)。"""
|
||||
|
||||
max_facts: int = Field(default=100, ge=10, le=500, description="最多保留的事实条数")
|
||||
fact_confidence_threshold: float = Field(default=0.7, ge=0.0, le=1.0, description="事实置信度阈值,低于此值不保存")
|
||||
debounce_seconds: int = Field(default=30, ge=5, le=300, description="对话结束后延迟多少秒才触发 LLM 提取")
|
||||
model_name: str = Field(default="", description="用于事实提取的 LLM 模型名;空则继承全局默认模型")
|
||||
token_budget: int = Field(default=2000, ge=200, le=8000, description="注入系统提示词的 token 预算")
|
||||
|
||||
|
||||
class MemoryConfig(BaseModel):
|
||||
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="记忆子系统总开关")
|
||||
version: str = Field(
|
||||
default="v2",
|
||||
description="记忆系统版本:v1(原版 DeerFlow,memory.json + LLM 自动提取) 或 v2(USER.md/MEMORY.md)",
|
||||
)
|
||||
provider: str = Field(
|
||||
default="builtin",
|
||||
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
|
||||
)
|
||||
|
||||
v1: V1MemoryConfig = Field(default_factory=V1MemoryConfig)
|
||||
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
|
||||
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
|
||||
injection: InjectionConfig = Field(default_factory=InjectionConfig)
|
||||
@@ -129,29 +144,26 @@ class MemoryConfig(BaseModel):
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# per-user 运行时覆盖
|
||||
# per-user 运行时覆盖 — 白名单
|
||||
# ===========================================================================
|
||||
|
||||
# 用户可在运行时覆盖的字段白名单 (点号路径)。
|
||||
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
|
||||
# 用户唯一可覆盖的字段:召回深度。
|
||||
# injection.* / builtin.* / enabled 始终由 config.yaml 控制,用户不可关闭。
|
||||
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"injection.enabled",
|
||||
"injection.max_tokens",
|
||||
"injection.include_builtin",
|
||||
"injection.include_hindsight",
|
||||
"hindsight.memory_mode",
|
||||
"hindsight.recall_budget",
|
||||
"hindsight.bank_mission",
|
||||
}
|
||||
)
|
||||
|
||||
_VALID_RECALL_BUDGETS: frozenset[str] = frozenset({"low", "mid", "high"})
|
||||
|
||||
|
||||
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
|
||||
|
||||
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
|
||||
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
|
||||
输入支持嵌套形式 ``{"hindsight": {"recall_budget": "high"}}`` 或
|
||||
点号扁平形式 ``{"hindsight.recall_budget": "high"}``。
|
||||
非白名单字段被丢弃并 log。
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
@@ -182,21 +194,6 @@ def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any
|
||||
return merged
|
||||
|
||||
|
||||
def _load_json_file(path: Any) -> dict[str, Any]:
|
||||
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return {}
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 全局单例 + 加载入口
|
||||
# ===========================================================================
|
||||
@@ -221,85 +218,92 @@ def load_memory_config_from_dict(config_dict: dict) -> None:
|
||||
_memory_config = MemoryConfig(**config_dict)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# per-user recall_budget 进程内 cache
|
||||
#
|
||||
# Gateway lifespan 启动时调用 load_recall_budget_cache() 从 DB 批量加载。
|
||||
# API 写入时同步更新 cache,使 get_effective_memory_config() 保持同步读取。
|
||||
# ===========================================================================
|
||||
|
||||
_recall_budget_cache: dict[str, str] = {}
|
||||
|
||||
|
||||
def load_recall_budget_cache(data: dict[str, str]) -> None:
|
||||
"""用 DB 批量加载结果替换整个 cache (启动时调用一次)。"""
|
||||
global _recall_budget_cache
|
||||
_recall_budget_cache = dict(data)
|
||||
logger.info("recall_budget cache loaded: %d user(s)", len(_recall_budget_cache))
|
||||
|
||||
|
||||
def set_recall_budget_cache(user_id: str, value: str) -> None:
|
||||
"""写入单个用户的 recall_budget cache。"""
|
||||
_recall_budget_cache[user_id] = value
|
||||
|
||||
|
||||
def delete_recall_budget_cache(user_id: str) -> None:
|
||||
"""从 cache 中移除某用户的 recall_budget。"""
|
||||
_recall_budget_cache.pop(user_id, None)
|
||||
|
||||
|
||||
def get_recall_budget_from_cache(user_id: str) -> str | None:
|
||||
"""返回 cache 中某用户的 recall_budget,未设置时返回 None。"""
|
||||
return _recall_budget_cache.get(user_id)
|
||||
|
||||
|
||||
def get_effective_memory_config(user_id: str) -> MemoryConfig:
|
||||
"""按三层覆盖优先级,返回某用户的生效记忆配置。
|
||||
"""返回某用户的生效记忆配置 (同步,从进程内 cache 读取 per-user 覆盖)。
|
||||
|
||||
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
|
||||
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
|
||||
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
|
||||
优先级 (高 -> 低):per-user DB 覆盖 (经 cache) > config.yaml > 默认值。
|
||||
有 per-user 记录时自动强制 enabled=True。
|
||||
"""
|
||||
base = _memory_config.model_dump()
|
||||
|
||||
recall_budget = _recall_budget_cache.get(user_id)
|
||||
if recall_budget is not None:
|
||||
base.setdefault("hindsight", {})["recall_budget"] = recall_budget
|
||||
base["enabled"] = True
|
||||
try:
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
paths = get_paths()
|
||||
global_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.memory_global_overrides_file)
|
||||
)
|
||||
user_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.user_memory_config_file(user_id))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
|
||||
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
|
||||
try:
|
||||
return MemoryConfig(**merged)
|
||||
return MemoryConfig(**base)
|
||||
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
|
||||
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
|
||||
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
|
||||
# ===========================================================================
|
||||
# Gateway 调用的异步 DB 对接函数
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
async def save_user_memory_override_db(user_id: str, patch: dict[str, Any], store: Any) -> dict[str, Any]:
|
||||
"""把白名单字段写入 DB store 并更新 cache,返回生效配置 dict。
|
||||
|
||||
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
|
||||
``store`` 为 ``UserMemoryConfigStore`` 实例 (避免循环 import,用 Any)。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
current = _load_json_file(path)
|
||||
filtered = _filter_to_whitelist(patch)
|
||||
merged = _deep_merge(current, filtered)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return merged
|
||||
recall_budget = (filtered.get("hindsight") or {}).get("recall_budget")
|
||||
if recall_budget is not None:
|
||||
if recall_budget not in _VALID_RECALL_BUDGETS:
|
||||
raise ValueError(f"recall_budget 必须为 {sorted(_VALID_RECALL_BUDGETS)} 之一,收到:{recall_budget!r}")
|
||||
await store.set_recall_budget(user_id, recall_budget)
|
||||
set_recall_budget_cache(user_id, recall_budget)
|
||||
return get_effective_memory_config(user_id).model_dump()
|
||||
|
||||
|
||||
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
|
||||
"""撤销某用户的 per-user 覆盖。
|
||||
async def delete_user_memory_override_db(user_id: str, field: str | None, store: Any) -> dict[str, Any]:
|
||||
"""撤销某用户的 per-user 覆盖并更新 cache,返回生效配置 dict。
|
||||
|
||||
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
|
||||
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
|
||||
``field=None`` 清空全部;``field="hindsight.recall_budget"`` 只清该字段。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
if field is None:
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return {}
|
||||
|
||||
current = _load_json_file(path)
|
||||
if "." in field:
|
||||
section, sub_field = field.split(".", 1)
|
||||
if isinstance(current.get(section), dict):
|
||||
current[section].pop(sub_field, None)
|
||||
if not current[section]:
|
||||
current.pop(section, None)
|
||||
if field is None or field == "hindsight.recall_budget":
|
||||
await store.delete(user_id)
|
||||
delete_recall_budget_cache(user_id)
|
||||
else:
|
||||
current.pop(field, None)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return current
|
||||
logger.warning("忽略删除非白名单的记忆配置覆盖字段: %s", field)
|
||||
return get_effective_memory_config(user_id).model_dump()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"V1MemoryConfig",
|
||||
"BuiltinMemoryConfig",
|
||||
"HindsightMemoryConfig",
|
||||
"InjectionConfig",
|
||||
@@ -310,6 +314,10 @@ __all__ = [
|
||||
"set_memory_config",
|
||||
"load_memory_config_from_dict",
|
||||
"get_effective_memory_config",
|
||||
"save_user_memory_override",
|
||||
"delete_user_memory_override",
|
||||
"load_recall_budget_cache",
|
||||
"set_recall_budget_cache",
|
||||
"delete_recall_budget_cache",
|
||||
"get_recall_budget_from_cache",
|
||||
"save_user_memory_override_db",
|
||||
"delete_user_memory_override_db",
|
||||
]
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Configuration for skill stop middleware."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SkillStopConfig(BaseModel):
|
||||
"""Configuration for the skill stop middleware.
|
||||
|
||||
When enabled, the middleware will intercept skill invocations and stop
|
||||
execution when a matching skill is about to be used.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable skill stop middleware. When True, execution will stop when specified skills are invoked.",
|
||||
)
|
||||
|
||||
skill_names: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="List of skill names to intercept. When the model reads a file from these skill directories, execution will stop.",
|
||||
)
|
||||
|
||||
stop_message: str | None = Field(
|
||||
default=None,
|
||||
description="Custom message to display when stopping. If None, a default message is used.",
|
||||
)
|
||||
|
||||
stop_on_match: bool = Field(
|
||||
default=True,
|
||||
description="If True, stop execution when skill matches. If False, only log the match without stopping.",
|
||||
)
|
||||
@@ -6,6 +6,18 @@ from pydantic import BaseModel, Field
|
||||
from deerflow.config.runtime_paths import project_root, resolve_path
|
||||
|
||||
|
||||
class SecurityScanConfig(BaseModel):
|
||||
"""Toggle for the LLM-based skill security scanner."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"When false, skip the LLM security scan applied to skill installs, "
|
||||
"custom-skill edits, and rollbacks. The scan is then treated as 'allow'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SkillsConfig(BaseModel):
|
||||
"""Configuration for skills system"""
|
||||
|
||||
@@ -21,6 +33,10 @@ class SkillsConfig(BaseModel):
|
||||
default="/mnt/skills",
|
||||
description="Path where skills are mounted in the sandbox container",
|
||||
)
|
||||
security_scan: SecurityScanConfig = Field(
|
||||
default_factory=SecurityScanConfig,
|
||||
description="LLM security-scan toggle for skill content.",
|
||||
)
|
||||
|
||||
def get_skills_path(self) -> Path:
|
||||
"""
|
||||
|
||||
@@ -61,3 +61,23 @@ class AgentStore(ABC):
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
"""Return the number of agents owned by ``user_id`` (user_id IS NOT NULL)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
"""Set or clear the homepage-selector featured rank for an agent.
|
||||
|
||||
Returns the updated row, or ``None`` if the agent does not exist.
|
||||
Caller is responsible for authorization (admin) and for verifying
|
||||
that ``published`` is true when ``order`` is not ``None``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
|
||||
raise NotImplementedError
|
||||
+21
@@ -55,6 +55,7 @@ class MemoryAgentStore(AgentStore):
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
@@ -70,6 +71,7 @@ class MemoryAgentStore(AgentStore):
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -117,3 +119,22 @@ class MemoryAgentStore(AgentStore):
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
+7
-1
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy import BigInteger, Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -21,5 +21,11 @@ class AgentRow(Base):
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
# frontend can seed initial ranks from ``Date.now()`` (ms-resolution
|
||||
# timestamps overflow a 4-byte INT on MySQL).
|
||||
featured_order: Mapped[int | None] = mapped_column(BigInteger, nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -142,3 +142,35 @@ class AgentRepository(AgentStore):
|
||||
stmt = select(func.count()).select_from(AgentRow).where(AgentRow.user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
return await session.scalar(stmt) or 0
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row.featured_order = order
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(AgentRow)
|
||||
.where(AgentRow.featured_order.is_not(None), self._visible_filter(user_id))
|
||||
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
if not agent_ids:
|
||||
return []
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._visible_filter(user_id))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars())
|
||||
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
+36
-1
@@ -8,4 +8,39 @@ def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository
|
||||
return AIWritingSessionRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
|
||||
# ── 进程级默认 repo 单例 ──────────────────────────────────────────────────────
|
||||
#
|
||||
# AI 写作 graph 注册成 LangGraph Server assistant 后,节点跑在 harness 进程里,
|
||||
# 拿不到 FastAPI 的 ``request.app.state``;harness → app 又是禁止的 import 方向。
|
||||
# 这里提供一个 app 启动时注册、harness 节点运行时读取的「跨边界单例」通道。
|
||||
#
|
||||
# 写入:app/gateway/deps.py 在创建完 session_factory 之后调
|
||||
# ``register_default_store(app.state.ai_writing_session_store)``
|
||||
# 读取:节点里 ``get_default_store()``,None 表示未配置 DB(跳过写入即可)。
|
||||
_default_store: AIWritingSessionRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: AIWritingSessionRepository | None) -> None:
|
||||
"""登记进程级默认 AI 写作 session store。
|
||||
|
||||
幂等:可多次调用,覆盖前值。传 ``None`` 用于显式清理(主要给测试)。
|
||||
"""
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> AIWritingSessionRepository | None:
|
||||
"""返回进程级默认 AI 写作 session store;未配置时为 ``None``。
|
||||
|
||||
节点调用方应在拿到 ``None`` 时跳过 DB 写入(写入是「锦上添花」非致命)。
|
||||
"""
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AIWritingSessionRow",
|
||||
"AIWritingSessionRepository",
|
||||
"make_ai_writing_session_store",
|
||||
"register_default_store",
|
||||
"get_default_store",
|
||||
]
|
||||
+15
-5
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy import String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class AIWritingSessionRow(Base):
|
||||
@@ -22,7 +23,16 @@ class AIWritingSessionRow(Base):
|
||||
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
|
||||
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
|
||||
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
|
||||
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
|
||||
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
# 时间列必须用 BeijingDateTime(项目规范,与 threads_meta 等其它表对齐)—— MySQL
|
||||
# DATETIME 不存时区,raw ``DateTime(timezone=True)`` 会把 UTC wall clock 写入
|
||||
# 列,读出来比真实北京时间晚 8 小时。``BeijingDateTime`` 在 bind 时 UTC→Beijing、
|
||||
# 在 result 时 attach +08:00,前后端拿到的就是北京时间且 tz-aware。
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+82
-1
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
@@ -63,6 +63,52 @@ class AIWritingSessionRepository:
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def upsert_session(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
title: str,
|
||||
user_intent: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
status: str = "in_progress",
|
||||
) -> dict:
|
||||
"""幂等:行不存在则创建,行存在则保留原数据(不覆盖 draft / review 等已有写入)。
|
||||
|
||||
AI 写作 graph 的 ``intent_parser_node`` 在每次进入时调一次(首启 + resume 时
|
||||
都会跑),所以必须幂等。resume 时若直接 ``create`` 会因主键冲突抛错;用
|
||||
``update`` 又找不到行而 no-op。这个方法把两条路合一:行不存在新建,行存在
|
||||
只刷新 ``updated_at`` 以便列表页排序前置。
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.upsert_session")
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, id)
|
||||
if row is None:
|
||||
row = AIWritingSessionRow(
|
||||
id=id,
|
||||
user_id=resolved_user_id,
|
||||
title=title,
|
||||
user_intent=user_intent,
|
||||
status=status,
|
||||
draft_title=None,
|
||||
draft_markdown=None,
|
||||
completed_interventions=None,
|
||||
review_result=None,
|
||||
transcript=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
# 行已存在(resume 路径):仅刷新 updated_at,保留 user_id / title / draft / review 等。
|
||||
# 若 user_id 之前为空(migration 兼容场景),用解析出来的填上。
|
||||
if row.user_id is None and resolved_user_id is not None:
|
||||
row.user_id = resolved_user_id
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
session_id: str,
|
||||
@@ -160,3 +206,38 @@ class AIWritingSessionRepository:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_older_than(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
only_finished: bool = False,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
|
||||
|
||||
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
|
||||
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
|
||||
|
||||
Args:
|
||||
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
|
||||
only_finished: True 时只清 status in (done/error),False 不看状态
|
||||
limit: 单批最多返回多少条,用于分批处理避免大事务
|
||||
"""
|
||||
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
|
||||
if only_finished:
|
||||
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
async def delete_by_ids(self, session_ids: list[str]) -> int:
|
||||
"""按 id 列表批量删除。返回实际删除的行数。"""
|
||||
if not session_ids:
|
||||
return 0
|
||||
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
@@ -10,12 +10,16 @@ None and fall back to in-memory implementations.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
@@ -26,36 +30,109 @@ def _json_serializer(obj: object) -> str:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
async def _run_alembic_upgrade(url: str) -> None:
|
||||
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
|
||||
|
||||
Runs after ``Base.metadata.create_all`` so brand-new deployments still get
|
||||
a complete schema even without Alembic, while existing deployments pick up
|
||||
in-place schema fixes (column widenings, new indexes, etc.) at the next
|
||||
restart — no manual ``alembic upgrade`` step required.
|
||||
|
||||
Notes:
|
||||
|
||||
* Each historical migration in ``migrations/versions/`` already guards
|
||||
itself with ``_has_table`` / ``_has_column``, so re-running them against
|
||||
a database that was originally built via ``create_all`` is a sequence of
|
||||
no-ops up to the first migration that actually has work to do.
|
||||
* The Alembic ``env.py`` calls ``asyncio.run`` internally, which cannot run
|
||||
inside an already-running loop — so we hop into a worker thread.
|
||||
* Failures here are logged but do not abort startup. The app remains
|
||||
operable on the ``create_all`` schema; the operator just won't get
|
||||
in-place fixes until they investigate.
|
||||
"""
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in ("1", "true", "yes"):
|
||||
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations (schema stays at create_all)")
|
||||
return
|
||||
|
||||
ini_path = Path(__file__).parent / "migrations" / "alembic.ini"
|
||||
if not ini_path.exists():
|
||||
logger.warning("Alembic config not found at %s; skipping auto-migrate", ini_path)
|
||||
return
|
||||
|
||||
# Alembic's ``Config`` is backed by ConfigParser, which performs ``%``
|
||||
# interpolation on every option value. URL-encoded credentials such as
|
||||
# ``kland%4088124932`` (the ``@`` in ``kland@88124932``) trip that up with
|
||||
# ``ValueError: invalid interpolation syntax ... at position N``. Doubling
|
||||
# the percent signs neutralises the interpolation; ConfigParser un-doubles
|
||||
# them back to the original value when read.
|
||||
escaped_url = url.replace("%", "%%")
|
||||
|
||||
def _do_upgrade() -> None:
|
||||
# Import inside the worker so a missing Alembic install can't break
|
||||
# the engine import chain.
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
cfg = Config(str(ini_path))
|
||||
cfg.set_main_option("sqlalchemy.url", escaped_url)
|
||||
# The ``script_location`` in alembic.ini is ``%(here)s`` which points
|
||||
# at the directory of alembic.ini itself — already correct for us.
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
# Bound the upgrade so a stuck DDL (e.g. MySQL metadata-lock wait) cannot
|
||||
# hang startup forever. Tunable via DEER_FLOW_ALEMBIC_TIMEOUT (seconds).
|
||||
# NOTE: asyncio.wait_for cancels the *await* on timeout; the worker thread
|
||||
# itself keeps running (Python can't kill threads). That's acceptable —
|
||||
# the orphan thread will exit when its MySQL connection eventually errors
|
||||
# or when the process restarts. The point here is to unblock startup.
|
||||
try:
|
||||
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
|
||||
except ValueError:
|
||||
timeout_s = 60.0
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s)
|
||||
logger.info("Alembic migrations applied (or already up-to-date)")
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(
|
||||
"Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.",
|
||||
timeout_s,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
@@ -79,11 +156,50 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def init_engine(
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def _auto_create_mysql_db(url: str) -> None:
|
||||
"""Create the target MySQL database when it is missing.
|
||||
|
||||
``create_all`` can create tables, but MySQL requires the database/schema to
|
||||
exist before the application URL can connect. We connect to the built-in
|
||||
``mysql`` maintenance database on the same server and issue a conservative
|
||||
``CREATE DATABASE IF NOT EXISTS``.
|
||||
"""
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
|
||||
|
||||
maint_url = parsed.set(database="mysql")
|
||||
maint_engine = create_async_engine(
|
||||
maint_url.render_as_string(hide_password=False),
|
||||
isolation_level="AUTOCOMMIT",
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
quoted = "`" + db_name.replace("`", "``") + "`"
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
f"CREATE DATABASE IF NOT EXISTS {quoted} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||||
)
|
||||
)
|
||||
logger.info("Auto-created MySQL database if missing: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
url: str = "",
|
||||
@@ -94,8 +210,8 @@ async def init_engine(
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Postgres connection pool size.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
@@ -106,20 +222,20 @@ async def init_engine(
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
@@ -150,29 +266,31 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Auto-create tables (dev convenience). Production should use Alembic.
|
||||
# Auto-create tables for fresh deployments, then let Alembic apply any
|
||||
# in-place schema fixes on top (see ``_run_alembic_upgrade`` below). Both
|
||||
# are idempotent so repeating them across restarts is harmless.
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
# Import all models so Base.metadata discovers them.
|
||||
@@ -184,21 +302,41 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s", backend)
|
||||
|
||||
@@ -209,10 +347,10 @@ async def init_engine_from_config(config) -> None:
|
||||
await init_engine("memory")
|
||||
return
|
||||
await init_engine(
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Per-user memory configuration override persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
|
||||
from deerflow.persistence.memory_config.memory import MemoryUserMemoryConfigStore
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.memory_config.sql import SqlUserMemoryConfigRepository
|
||||
|
||||
|
||||
def make_user_memory_config_store(session_factory) -> UserMemoryConfigStore:
|
||||
if session_factory is None:
|
||||
return MemoryUserMemoryConfigStore()
|
||||
return SqlUserMemoryConfigRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"UserMemoryConfigStore",
|
||||
"UserMemoryConfigRow",
|
||||
"MemoryUserMemoryConfigStore",
|
||||
"SqlUserMemoryConfigRepository",
|
||||
"make_user_memory_config_store",
|
||||
]
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"""Abstract persistence API for per-user memory config overrides."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class UserMemoryConfigStore(ABC):
|
||||
|
||||
@abstractmethod
|
||||
async def get_recall_budget(self, user_id: str) -> str | None:
|
||||
"""Return the user's recall_budget override, or None if not set."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_recall_budget(self, user_id: str, value: str) -> None:
|
||||
"""Upsert the user's recall_budget override."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, user_id: str) -> None:
|
||||
"""Remove all per-user overrides for the given user."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def load_all(self) -> dict[str, str]:
|
||||
"""Bulk-load all user recall_budget overrides.
|
||||
|
||||
Returns {user_id: recall_budget} for all rows where recall_budget IS NOT NULL.
|
||||
Called once at startup to warm the in-memory cache.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""In-memory user memory config store (used when database.backend=memory)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
|
||||
|
||||
|
||||
class MemoryUserMemoryConfigStore(UserMemoryConfigStore):
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._data: dict[str, str] = {}
|
||||
|
||||
async def get_recall_budget(self, user_id: str) -> str | None:
|
||||
return self._data.get(user_id)
|
||||
|
||||
async def set_recall_budget(self, user_id: str, value: str) -> None:
|
||||
self._data[user_id] = value
|
||||
|
||||
async def delete(self, user_id: str) -> None:
|
||||
self._data.pop(user_id, None)
|
||||
|
||||
async def load_all(self) -> dict[str, str]:
|
||||
return dict(self._data)
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
"""ORM model for per-user memory configuration overrides."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class UserMemoryConfigRow(Base):
|
||||
__tablename__ = "user_memory_configs"
|
||||
|
||||
# One row per user; upserted on write.
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
|
||||
# The only user-writable field. NULL means "use global default from config.yaml".
|
||||
# Allowed values: "low" | "mid" | "high".
|
||||
recall_budget: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""SQLAlchemy-backed user memory config repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
|
||||
|
||||
class SqlUserMemoryConfigRepository(UserMemoryConfigStore):
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_recall_budget(self, user_id: str) -> str | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMemoryConfigRow, user_id)
|
||||
return row.recall_budget if row is not None else None
|
||||
|
||||
async def set_recall_budget(self, user_id: str, value: str) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMemoryConfigRow, user_id)
|
||||
if row is None:
|
||||
row = UserMemoryConfigRow(
|
||||
user_id=user_id,
|
||||
recall_budget=value,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.recall_budget = value
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
|
||||
async def delete(self, user_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMemoryConfigRow, user_id)
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
|
||||
async def load_all(self) -> dict[str, str]:
|
||||
stmt = select(UserMemoryConfigRow).where(UserMemoryConfigRow.recall_budget.is_not(None))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row.user_id: row.recall_budget for row in result.scalars() if row.recall_budget is not None}
|
||||
+23
-7
@@ -9,9 +9,11 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -44,13 +46,27 @@ def run_migrations_offline() -> None:
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
|
||||
# On MySQL, shrink the session lock waits so a blocked DDL fails fast
|
||||
# instead of hanging on the server default (lock_wait_timeout = 31536000s
|
||||
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
|
||||
# transaction on another connection can freeze startup indefinitely.
|
||||
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
|
||||
if connection.dialect.name == "mysql":
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
|
||||
|
||||
The ``transcript`` column stores the full AI-writing conversation timeline as a
|
||||
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
|
||||
a transcript with many progress events overflows it and the save endpoint
|
||||
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
|
||||
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
|
||||
|
||||
Revision ID: 20260519_02
|
||||
Revises: 20260519_01
|
||||
Create Date: 2026-05-19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
|
||||
|
||||
revision = "20260519_02"
|
||||
down_revision = "20260519_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=LONGTEXT(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
|
||||
# is a developer-only path.
|
||||
import sqlalchemy as sa
|
||||
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
"""Homepage agent selector: ``user_agent_pin`` table + ``agents.featured_order``.
|
||||
|
||||
Adds two pieces backing the new chat-homepage agent picker:
|
||||
|
||||
- ``agents.featured_order`` (nullable INT) — admin-curated featured rank.
|
||||
NULL = not shown on the selector. Non-NULL rows sort ascending.
|
||||
- ``user_agent_pin`` — per-user pinned agents (user's "favorites"). Composite
|
||||
PK ``(user_id, agent_id)`` so a user pins each agent at most once. Foreign
|
||||
key to ``agents.id`` with ``ON DELETE CASCADE`` so deleting an agent also
|
||||
removes every user's pin of it.
|
||||
|
||||
Revision ID: 20260520_01
|
||||
Revises: 20260519_02
|
||||
Create Date: 2026-05-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260520_01"
|
||||
down_revision = "20260519_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. agents.featured_order — ALTER existing table. Skipped on brand-new DBs
|
||||
# where create_all already materialised the column.
|
||||
if _has_table("agents") and not _has_column("agents", "featured_order"):
|
||||
op.add_column("agents", sa.Column("featured_order", sa.Integer(), nullable=True))
|
||||
if _has_table("agents") and not _has_index("agents", "ix_agents_featured_order"):
|
||||
op.create_index("ix_agents_featured_order", "agents", ["featured_order"])
|
||||
|
||||
# 2. user_agent_pin — per-user pinned agents.
|
||||
if not _has_table("user_agent_pin"):
|
||||
op.create_table(
|
||||
"user_agent_pin",
|
||||
sa.Column("user_id", sa.String(64), nullable=False),
|
||||
sa.Column(
|
||||
"agent_id",
|
||||
sa.String(64),
|
||||
sa.ForeignKey("agents.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("user_id", "agent_id", name="pk_user_agent_pin"),
|
||||
)
|
||||
op.create_index("ix_user_agent_pin_sort_order", "user_agent_pin", ["sort_order"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_index("user_agent_pin", "ix_user_agent_pin_sort_order"):
|
||||
op.drop_index("ix_user_agent_pin_sort_order", table_name="user_agent_pin")
|
||||
if _has_table("user_agent_pin"):
|
||||
op.drop_table("user_agent_pin")
|
||||
if _has_index("agents", "ix_agents_featured_order"):
|
||||
op.drop_index("ix_agents_featured_order", table_name="agents")
|
||||
if _has_column("agents", "featured_order"):
|
||||
op.drop_column("agents", "featured_order")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Widen ``agents.featured_order`` from INT to BIGINT.
|
||||
|
||||
The previous revision (``20260520_01``) created ``featured_order`` as plain
|
||||
``INTEGER``. The frontend seeds the initial rank with ``Date.now()`` so admins
|
||||
get "newest featured sinks to the bottom" with zero UI — but ms-resolution
|
||||
timestamps (~1.78e12) overflow a 4-byte signed INT on MySQL, surfacing as
|
||||
|
||||
sqlalchemy.exc.DataError: (1264, "Out of range value for column 'featured_order'")
|
||||
|
||||
SQLite and PostgreSQL ``INTEGER`` is already 64-bit / arbitrary-precision and
|
||||
needs no change, but the migration runs everywhere for schema parity.
|
||||
|
||||
Revision ID: 20260520_02
|
||||
Revises: 20260520_01
|
||||
Create Date: 2026-05-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260520_02"
|
||||
down_revision = "20260520_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("agents", "featured_order"):
|
||||
return
|
||||
bind = op.get_bind()
|
||||
# SQLite has only INTEGER (dynamic precision) — alter_column would rebuild
|
||||
# the table for nothing. PostgreSQL INTEGER is 32-bit too, so alter it.
|
||||
if bind.dialect.name == "sqlite":
|
||||
return
|
||||
op.alter_column(
|
||||
"agents",
|
||||
"featured_order",
|
||||
type_=sa.BigInteger(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_column("agents", "featured_order"):
|
||||
return
|
||||
bind = op.get_bind()
|
||||
if bind.dialect.name == "sqlite":
|
||||
return
|
||||
# Any rows with values > 2^31-1 would be truncated. Downgrade is dev-only.
|
||||
op.alter_column(
|
||||
"agents",
|
||||
"featured_order",
|
||||
type_=sa.Integer(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Drop ``user_agent_pin`` (homepage selector simplified to admin-only).
|
||||
|
||||
The original design split the homepage agent picker into two buckets:
|
||||
admin-curated "featured" + per-user "pinned". Product feedback showed the
|
||||
per-user side added complexity without clear value — admins now own the
|
||||
single list shown to everyone. ``agents.featured_order`` is the only piece
|
||||
that survives; this revision removes the now-unused pin table.
|
||||
|
||||
Revision ID: 20260520_03
|
||||
Revises: 20260520_02
|
||||
Create Date: 2026-05-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260520_03"
|
||||
down_revision = "20260520_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_index("user_agent_pin", "ix_user_agent_pin_sort_order"):
|
||||
op.drop_index("ix_user_agent_pin_sort_order", table_name="user_agent_pin")
|
||||
if _has_table("user_agent_pin"):
|
||||
op.drop_table("user_agent_pin")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table("user_agent_pin"):
|
||||
op.create_table(
|
||||
"user_agent_pin",
|
||||
sa.Column("user_id", sa.String(64), nullable=False),
|
||||
sa.Column(
|
||||
"agent_id",
|
||||
sa.String(64),
|
||||
sa.ForeignKey("agents.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("user_id", "agent_id", name="pk_user_agent_pin"),
|
||||
)
|
||||
op.create_index("ix_user_agent_pin_sort_order", "user_agent_pin", ["sort_order"])
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Create user_memory_configs table.
|
||||
|
||||
Replaces the per-user file-based memory config override
|
||||
({base_dir}/users/{user_id}/memory_config.json) with a DB row.
|
||||
Stores only hindsight.recall_budget — the single user-writable field
|
||||
after the memory config simplification.
|
||||
|
||||
Revision ID: 20260521_01
|
||||
Revises: 20260519_02
|
||||
Create Date: 2026-05-21
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260521_01"
|
||||
down_revision = "20260519_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("user_memory_configs"):
|
||||
return
|
||||
op.create_table(
|
||||
"user_memory_configs",
|
||||
sa.Column("user_id", sa.String(64), primary_key=True),
|
||||
sa.Column("recall_budget", sa.String(16), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("user_memory_configs"):
|
||||
op.drop_table("user_memory_configs")
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""Merge agent pin/featured and user memory config migration heads.
|
||||
|
||||
Revision ID: 20260521_02
|
||||
Revises: 20260520_03, 20260521_01
|
||||
Create Date: 2026-05-21
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
revision: str = "20260521_02"
|
||||
down_revision: str | tuple[str, str] = ("20260520_03", "20260521_01")
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""Add description column to scheduled_tasks.
|
||||
|
||||
Revision ID: 20260522_01
|
||||
Revises: 20260521_02
|
||||
Create Date: 2026-05-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260522_01"
|
||||
down_revision = "20260521_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_tasks", "description"):
|
||||
op.add_column(
|
||||
"scheduled_tasks",
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("scheduled_tasks", "description"):
|
||||
op.drop_column("scheduled_tasks", "description")
|
||||
+2
@@ -16,6 +16,7 @@ there is no matching entity directory.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.agents.model import AgentRow
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
@@ -55,5 +56,6 @@ __all__ = [
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"UserMemoryConfigRow",
|
||||
"UserRow",
|
||||
]
|
||||
+1
@@ -31,6 +31,7 @@ class ScheduledTaskRow(Base):
|
||||
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
prompt: Mapped[str] = mapped_column(Text)
|
||||
schedule_text: Mapped[str] = mapped_column(String(256))
|
||||
cron_expr: Mapped[str] = mapped_column(String(128))
|
||||
|
||||
+1
@@ -76,6 +76,7 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
source_agent_name=data.get("source_agent_name"),
|
||||
execution_agent_name=data.get("execution_agent_name"),
|
||||
name=data["name"],
|
||||
description=data.get("description"),
|
||||
prompt=data["prompt"],
|
||||
schedule_text=data["schedule_text"],
|
||||
cron_expr=data["cron_expr"],
|
||||
|
||||
+11
@@ -56,6 +56,17 @@ class ThreadMetaStore(abc.ABC):
|
||||
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
"""Backfill ``assistant_id`` for a thread row.
|
||||
|
||||
Used by ``start_run`` when the row was pre-created via ``POST /threads``
|
||||
without an assistant_id (the SDK's ``client.threads.create()`` doesn't
|
||||
pass it), and the first run on the thread reveals the assistant. No-op
|
||||
if the thread does not exist or the owner check fails.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
"""Merge ``metadata`` into the thread's metadata field.
|
||||
|
||||
+8
@@ -117,6 +117,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
|
||||
if record is None:
|
||||
return
|
||||
record["assistant_id"] = assistant_id
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
|
||||
if record is None:
|
||||
|
||||
+14
@@ -174,6 +174,20 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
|
||||
async def update_assistant_id(
|
||||
self,
|
||||
thread_id: str,
|
||||
assistant_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_assistant_id")
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
|
||||
async def update_metadata(
|
||||
self,
|
||||
thread_id: str,
|
||||
|
||||
@@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
|
||||
@@ -39,6 +40,25 @@ class PortableJSON(TypeDecorator):
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class PortableLongText(TypeDecorator):
|
||||
"""A large text column.
|
||||
|
||||
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
|
||||
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
|
||||
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
|
||||
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
|
||||
already unbounded.
|
||||
"""
|
||||
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(LONGTEXT())
|
||||
return dialect.type_descriptor(Text())
|
||||
|
||||
|
||||
class BeijingDateTime(TypeDecorator):
|
||||
"""Store timestamps as Beijing (UTC+8) wall-clock time.
|
||||
|
||||
|
||||
@@ -33,6 +33,13 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Flush the buffer at least this often (seconds) even when it never fills up.
|
||||
# Live consumers — notably the scheduled-task run-detail page, which polls
|
||||
# ``GET /runs/{rid}/messages`` every 4s — would otherwise see nothing until a
|
||||
# run ends, because a short run can finish before the buffer reaches
|
||||
# ``flush_threshold``.
|
||||
_FLUSH_INTERVAL_SECONDS = 3.0
|
||||
|
||||
|
||||
class RunJournal(BaseCallbackHandler):
|
||||
"""LangChain callback handler that captures events to RunEventStore."""
|
||||
@@ -56,6 +63,9 @@ class RunJournal(BaseCallbackHandler):
|
||||
# Write buffer
|
||||
self._buffer: list[dict] = []
|
||||
self._pending_flush_tasks: set[asyncio.Task[None]] = set()
|
||||
# Monotonic timestamp of the last flush attempt — drives time-based
|
||||
# flushing so incremental progress is visible to live pollers.
|
||||
self._last_flush_at: float = time.monotonic()
|
||||
|
||||
# Token accumulators
|
||||
self._total_input_tokens = 0
|
||||
@@ -269,8 +279,14 @@ class RunJournal(BaseCallbackHandler):
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
# Flush on a full buffer, or once enough time has elapsed since the
|
||||
# last flush. The time-based path lets live consumers (the
|
||||
# scheduled-task run-detail page polls every 4s) see incremental
|
||||
# progress even when a run ends before the buffer fills up.
|
||||
if len(self._buffer) >= self._flush_threshold:
|
||||
self._flush_sync()
|
||||
elif time.monotonic() - self._last_flush_at >= _FLUSH_INTERVAL_SECONDS:
|
||||
self._flush_sync()
|
||||
|
||||
def _flush_sync(self) -> None:
|
||||
"""Best-effort flush of buffer to RunEventStore.
|
||||
@@ -293,6 +309,7 @@ class RunJournal(BaseCallbackHandler):
|
||||
return
|
||||
batch = self._buffer.copy()
|
||||
self._buffer.clear()
|
||||
self._last_flush_at = time.monotonic()
|
||||
task = loop.create_task(self._flush_async(batch))
|
||||
self._pending_flush_tasks.add(task)
|
||||
task.add_done_callback(self._on_flush_done)
|
||||
@@ -307,8 +324,10 @@ class RunJournal(BaseCallbackHandler):
|
||||
self.run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
# Return failed events to buffer for retry on next flush
|
||||
# Return failed events to buffer for retry on next flush, then
|
||||
# re-raise so _on_flush_done can tell success from failure.
|
||||
self._buffer = batch + self._buffer
|
||||
raise
|
||||
|
||||
def _on_flush_done(self, task: asyncio.Task) -> None:
|
||||
self._pending_flush_tasks.discard(task)
|
||||
@@ -316,7 +335,16 @@ class RunJournal(BaseCallbackHandler):
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
# The batch was already returned to the buffer by _flush_async.
|
||||
# Don't retry here — that would busy-loop on a persistent DB
|
||||
# outage; the next _put (or the final flush()) retries instead.
|
||||
logger.warning("Journal flush task failed: %s", exc)
|
||||
return
|
||||
# Events may have accumulated while this flush was in flight — drain
|
||||
# them now so incremental progress keeps reaching the store without
|
||||
# waiting for the next _put to cross a threshold.
|
||||
if self._buffer:
|
||||
self._flush_sync()
|
||||
|
||||
def _identify_caller(self, tags: list[str] | None) -> str:
|
||||
_tags = tags or []
|
||||
|
||||
@@ -130,8 +130,16 @@ async def run_agent(
|
||||
stream_subgraphs: bool = False,
|
||||
interrupt_before: list[str] | Literal["*"] | None = None,
|
||||
interrupt_after: list[str] | Literal["*"] | None = None,
|
||||
command: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Execute an agent in the background, publishing events to *bridge*."""
|
||||
"""Execute an agent in the background, publishing events to *bridge*.
|
||||
|
||||
``command`` 来自 LangGraph SDK 的 ``Command`` 协议(``{resume, update, goto}``)。
|
||||
存在时把它翻译成 ``langgraph.types.Command`` 并作为 ``astream`` 的入参 ——
|
||||
Pregel 检查到 Command 输入会从最近 checkpoint 处恢复(典型场景:interrupt 之后
|
||||
的 ``Command(resume=user_input)``);不存在时仍按 ``graph_input`` dict 起新 run。
|
||||
AI 写作的 4 个干预卡就靠这个把 user_input 传回 ``pause_*_confirm`` 节点。
|
||||
"""
|
||||
|
||||
# Unpack infrastructure dependencies from RunContext.
|
||||
checkpointer = ctx.checkpointer
|
||||
@@ -269,10 +277,23 @@ async def run_agent(
|
||||
logger.info("Run %s: streaming with modes %s (requested: %s)", run_id, lg_modes, requested_modes)
|
||||
|
||||
# 7. Stream using graph.astream
|
||||
# 如果客户端传了 Command(典型:interrupt 后续提的 ``Command(resume=...)``),
|
||||
# 把它翻译成 langgraph.types.Command 作为 astream 输入;否则按 graph_input dict
|
||||
# 起新一轮。Pregel 检查到 Command 会从最近 checkpoint 处恢复,而不是重头跑。
|
||||
stream_input: Any = graph_input
|
||||
if command is not None:
|
||||
from langgraph.types import Command as _LGCommand
|
||||
|
||||
stream_input = _LGCommand(
|
||||
resume=command.get("resume"),
|
||||
update=command.get("update"),
|
||||
goto=command.get("goto"),
|
||||
)
|
||||
|
||||
if len(lg_modes) == 1 and not stream_subgraphs:
|
||||
# Single mode, no subgraphs: astream yields raw chunks
|
||||
single_mode = lg_modes[0]
|
||||
async for chunk in agent.astream(graph_input, config=runnable_config, stream_mode=single_mode):
|
||||
async for chunk in agent.astream(stream_input, config=runnable_config, stream_mode=single_mode):
|
||||
if record.abort_event.is_set():
|
||||
logger.info("Run %s abort requested — stopping", run_id)
|
||||
break
|
||||
@@ -281,7 +302,7 @@ async def run_agent(
|
||||
else:
|
||||
# Multiple modes or subgraphs: astream yields tuples
|
||||
async for item in agent.astream(
|
||||
graph_input,
|
||||
stream_input,
|
||||
config=runnable_config,
|
||||
stream_mode=lg_modes,
|
||||
subgraphs=stream_subgraphs,
|
||||
|
||||
+94
-4
@@ -18,6 +18,7 @@ from zoneinfo import ZoneInfo
|
||||
from croniter import croniter
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
|
||||
from deerflow.config.agents_config import AGENT_ID_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
@@ -305,12 +306,51 @@ class ScheduledTaskService:
|
||||
"""
|
||||
run = await self.store.get_run(run_id, user_id)
|
||||
if run is None:
|
||||
# Either the run doesn't exist, or the caller is a subscriber, not owner.
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
raise PermissionError("Only the run owner may edit result files")
|
||||
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
|
||||
|
||||
async def admin_update_run_file(
|
||||
self,
|
||||
run_id: str,
|
||||
file_index: int,
|
||||
new_content: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Owner-less variant of :meth:`update_run_file` used by the public viewer.
|
||||
|
||||
The public scheduled-task page is unauthenticated, so it cannot supply a
|
||||
``user_id``. Mutations from that page route here instead. The disk-write
|
||||
and result-row update logic is identical to the owner path.
|
||||
"""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
|
||||
|
||||
async def admin_update_run_body(self, run_id: str, new_content: str) -> dict[str, Any]:
|
||||
"""Overwrite the inline Markdown body (``result.content``) of a run.
|
||||
|
||||
Used by the public scheduled-task viewer so anonymous editors can fix
|
||||
typos in the "正文" tab. Unlike :meth:`admin_update_run_file`, no disk
|
||||
artifact backs the body — it lives entirely in the run row.
|
||||
"""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
result = run.get("result") if isinstance(run.get("result"), dict) else {}
|
||||
new_result = {**result, "content": new_content}
|
||||
await self.store.update_run(run_id, {"result": new_result})
|
||||
return {"content": new_content}
|
||||
|
||||
async def _apply_run_file_edit(
|
||||
self,
|
||||
run: dict[str, Any],
|
||||
run_id: str,
|
||||
file_index: int,
|
||||
new_content: str,
|
||||
) -> dict[str, Any]:
|
||||
result = run.get("result") or {}
|
||||
files = result.get("files") if isinstance(result, dict) else None
|
||||
if not isinstance(files, list) or file_index < 0 or file_index >= len(files):
|
||||
@@ -406,6 +446,7 @@ class ScheduledTaskService:
|
||||
prompt: str,
|
||||
schedule: str,
|
||||
timezone: str | None = None,
|
||||
description: str | None = None,
|
||||
source_thread_id: str | None = None,
|
||||
source_agent_name: str | None = None,
|
||||
execution_agent_name: str | None = None,
|
||||
@@ -420,6 +461,7 @@ class ScheduledTaskService:
|
||||
context = dict(execution_context or {})
|
||||
if execution_agent_name:
|
||||
context["agent_name"] = execution_agent_name
|
||||
normalized_description = description.strip() if isinstance(description, str) else None
|
||||
task = await self.store.create_task(
|
||||
{
|
||||
"task_id": str(uuid.uuid4()),
|
||||
@@ -429,6 +471,7 @@ class ScheduledTaskService:
|
||||
"source_agent_name": source_agent_name,
|
||||
"execution_agent_name": execution_agent_name,
|
||||
"name": final_name,
|
||||
"description": normalized_description or None,
|
||||
"prompt": prompt.strip(),
|
||||
"schedule_text": schedule.strip(),
|
||||
"cron_expr": cron_expr,
|
||||
@@ -451,6 +494,15 @@ class ScheduledTaskService:
|
||||
for key in ("name", "prompt", "enabled", "execution_agent_name"):
|
||||
if key in data and data[key] is not None:
|
||||
update_data[key] = data[key]
|
||||
# Description is nullable — accept ``None`` (clear) and empty strings
|
||||
# (also clear) explicitly, so the field can be removed via PATCH.
|
||||
if "description" in data:
|
||||
raw = data["description"]
|
||||
if isinstance(raw, str):
|
||||
trimmed = raw.strip()
|
||||
update_data["description"] = trimmed or None
|
||||
else:
|
||||
update_data["description"] = None
|
||||
# Names are matched against the per-user uniqueness constraint, so keep
|
||||
# the stored value trimmed to match create_task's normalization.
|
||||
if isinstance(update_data.get("name"), str):
|
||||
@@ -661,7 +713,7 @@ class ScheduledTaskService:
|
||||
retry_count: int = 0,
|
||||
) -> None:
|
||||
"""Mark a run failed and reflect the outcome on the parent task."""
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"定时任务执行失败:{error}")
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
|
||||
@@ -819,8 +871,17 @@ class ScheduledTaskService:
|
||||
try:
|
||||
request = SimpleNamespace(app=self.app)
|
||||
execution_context = dict(task.get("execution_context") or {})
|
||||
if task.get("execution_agent_name"):
|
||||
execution_context["agent_name"] = task["execution_agent_name"]
|
||||
stored_agent_ref = task.get("execution_agent_name")
|
||||
if stored_agent_ref:
|
||||
resolved_agent_id = await self._resolve_agent_id(stored_agent_ref, task["user_id"])
|
||||
if resolved_agent_id:
|
||||
execution_context["agent_name"] = resolved_agent_id
|
||||
else:
|
||||
logger.warning(
|
||||
"Scheduled task %s references unknown agent %r; falling back to default lead agent",
|
||||
task.get("task_id"),
|
||||
stored_agent_ref,
|
||||
)
|
||||
execution_context.update(
|
||||
{
|
||||
"is_scheduled_run": True,
|
||||
@@ -845,6 +906,35 @@ class ScheduledTaskService:
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
async def _resolve_agent_id(self, stored_ref: str, user_id: str) -> str | None:
|
||||
"""Resolve a scheduled task's ``execution_agent_name`` to a stable agent id.
|
||||
|
||||
Older tasks (created before the frontend was fixed) stored the agent's
|
||||
Chinese **display name** here; the runtime validator only accepts ids
|
||||
matching ``^[A-Za-z0-9_-]+$``, so those rows would 422 forever. We
|
||||
gracefully recover by looking the stored value up first as an id, then
|
||||
as a display name. Returns ``None`` when nothing visible matches — the
|
||||
caller then falls back to the default lead agent rather than crashing.
|
||||
"""
|
||||
store = getattr(self.app.state, "agent_store", None)
|
||||
if store is None:
|
||||
return stored_ref if AGENT_ID_PATTERN.fullmatch(stored_ref) else None
|
||||
if AGENT_ID_PATTERN.fullmatch(stored_ref):
|
||||
try:
|
||||
row = await store.get_visible(stored_ref, user_id)
|
||||
except Exception:
|
||||
row = None
|
||||
if row is not None:
|
||||
return row["id"]
|
||||
try:
|
||||
agents = await store.list_visible(user_id)
|
||||
except Exception:
|
||||
return None
|
||||
for agent in agents:
|
||||
if agent.get("name") == stored_ref:
|
||||
return agent.get("id")
|
||||
return None
|
||||
|
||||
async def _extract_latest_ai_text(self, thread_id: str) -> str | None:
|
||||
checkpointer = self.app.state.checkpointer
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
|
||||
+23
-3
@@ -10,6 +10,7 @@ Consumers: ``deerflow.runtime.runs.worker`` (SSE publishing) and
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -35,6 +36,16 @@ def serialize_lc_object(obj: Any) -> Any:
|
||||
return obj.dict()
|
||||
except Exception:
|
||||
pass
|
||||
# Dataclass instances —— LangGraph 的 ``Interrupt`` 就是 dataclass 没有
|
||||
# model_dump 也没有 dict 方法;之前 fallback 到 ``str(obj)`` 会把
|
||||
# ``Interrupt(value=..., id='...')`` 序列化成字符串,下游 SDK
|
||||
# (``@langchain/langgraph-sdk``) 期待 dict 结构 → 直接丢弃 → AI 写作
|
||||
# 的 4 个干预卡永远不弹(参 ai-writing-优化.md §阶段 3 排查记录)。
|
||||
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
|
||||
try:
|
||||
return {k: serialize_lc_object(v) for k, v in dataclasses.asdict(obj).items()}
|
||||
except Exception:
|
||||
pass
|
||||
# Last resort
|
||||
try:
|
||||
return str(obj)
|
||||
@@ -45,12 +56,21 @@ def serialize_lc_object(obj: Any) -> Any:
|
||||
def serialize_channel_values(channel_values: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Serialize channel values, stripping internal LangGraph keys.
|
||||
|
||||
Internal keys like ``__pregel_*`` and ``__interrupt__`` are removed
|
||||
to match what the LangGraph Platform API returns.
|
||||
``__pregel_*`` 键始终剥离。``__interrupt__`` 仅在**为空时**剥离 —— 因为
|
||||
``@langchain/langgraph-sdk`` 的 ``useStream`` 只在 ``event === "values"``
|
||||
通道里读 ``data.__interrupt__`` 并通过 ``stream.interrupt`` 暴露给上层。
|
||||
AI 写作的 4 个干预卡(material/outline/draft/review)就是靠它触发的;
|
||||
继续过滤会让前端永远拿不到 interrupt。
|
||||
|
||||
主聊天不消费 ``stream.interrupt``,但若我们让一个空数组溜过去,SDK 会
|
||||
fallback 成 ``{when: "breakpoint"}`` 的伪 interrupt —— 所以仅在数组非空
|
||||
时保留,两侧都不破坏。
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in channel_values.items():
|
||||
if key.startswith("__pregel_") or key == "__interrupt__":
|
||||
if key.startswith("__pregel_"):
|
||||
continue
|
||||
if key == "__interrupt__" and not value:
|
||||
continue
|
||||
result[key] = serialize_lc_object(value)
|
||||
return result
|
||||
|
||||
+6
-1
@@ -39,6 +39,12 @@ def _extract_json_object(raw: str) -> dict | None:
|
||||
|
||||
async def scan_skill_content(content: str, *, executable: bool = False, location: str = SKILL_MD_FILE, app_config: AppConfig | None = None) -> ScanResult:
|
||||
"""Screen skill content before it is written to disk."""
|
||||
config = app_config or get_app_config()
|
||||
# Operator-level kill switch: when ``skills.security_scan.enabled`` is false,
|
||||
# bypass the LLM round-trip entirely and let the content through.
|
||||
if not config.skills.security_scan.enabled:
|
||||
return ScanResult("allow", "Security scan disabled by config.")
|
||||
|
||||
rubric = (
|
||||
"You are a security reviewer for AI agent skills. "
|
||||
"Classify the content as allow, warn, or block. "
|
||||
@@ -49,7 +55,6 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
|
||||
prompt = f"Location: {location}\nExecutable: {str(executable).lower()}\n\nReview this content:\n-----\n{content}\n-----"
|
||||
|
||||
try:
|
||||
config = app_config or get_app_config()
|
||||
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
|
||||
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
|
||||
model_name = config.skill_evolution.curator.model_name
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from .agent_orchestration_tool import agent_orchestration_tool
|
||||
from .clarification_tool import ask_clarification_tool
|
||||
from .memory_tool import memory_tool
|
||||
from .present_file_tool import present_file_tool
|
||||
@@ -10,6 +11,7 @@ __all__ = [
|
||||
"setup_agent",
|
||||
"present_file_tool",
|
||||
"ask_clarification_tool",
|
||||
"agent_orchestration_tool",
|
||||
"memory_tool",
|
||||
"scheduled_task_tool",
|
||||
"view_image_tool",
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
"""Placeholder tool that lets the roundtable leader agent dispatch sub-agents.
|
||||
|
||||
Background
|
||||
----------
|
||||
The multi-agent ``/api/multi-agent/run/stream`` endpoint (``app.gateway.routers.multi_agent``)
|
||||
drives a *leader* agent whose only job is to delegate work to seat-specific sub-agents.
|
||||
Its SOUL.md tells the model to "use the agent_orchestration skill, passing agent_name
|
||||
and task". With no real tool registered under that name, the model used to emit
|
||||
``{"name": "agent_orchestration", "args": {"agent_name": ..., "task": ...}}`` calls
|
||||
that the runtime rejected with "agent_orchestration is not a valid tool", driving it
|
||||
into an unproductive retry loop (a single leader call could exceed 10 000 SSE frames).
|
||||
|
||||
This module exposes a no-op tool with the exact signature the leader naturally
|
||||
emits. The actual dispatch is handled by ``SkillStopMiddleware`` which intercepts
|
||||
the call, halts the run, and lets ``multi_agent.py`` parse the AIMessage's
|
||||
``tool_calls`` to extract ``(agent_name, task)`` pairs for the frontend.
|
||||
"""
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
|
||||
@tool("agent_orchestration", parse_docstring=True)
|
||||
def agent_orchestration_tool(agent_name: str, task: str) -> str:
|
||||
"""Delegate a task to a named sub-agent in the multi-agent roundtable.
|
||||
|
||||
Use this tool ONLY when you are the roundtable coordinator and need to
|
||||
dispatch work to one of the configured seat agents. Each call delegates
|
||||
ONE task to ONE agent. Emit multiple calls in the same turn to dispatch
|
||||
several seats in a single round of the discussion.
|
||||
|
||||
The call is intercepted by the runtime — you do not get a real result
|
||||
back synchronously. The orchestrator will collect every dispatched
|
||||
``(agent_name, task)`` pair, run the corresponding sub-agents, then
|
||||
re-prompt you with a synthesis instruction once they finish.
|
||||
|
||||
Args:
|
||||
agent_name: Name of the sub-agent to dispatch. MUST exactly match one of
|
||||
the names listed in your SOUL prompt's ``可调度智能体`` section.
|
||||
task: The instruction to hand to that sub-agent. Be specific — the
|
||||
sub-agent only sees this string (plus its own SOUL prompt), not the
|
||||
broader conversation context.
|
||||
"""
|
||||
# Real handling is in SkillStopMiddleware + the multi_agent router. This
|
||||
# body only runs as a fallback if the middleware is not configured; it
|
||||
# returns a placeholder so the model does not see a "not a valid tool" error.
|
||||
return f"dispatched:{agent_name}"
|
||||
+20
-7
@@ -36,18 +36,31 @@ def ask_clarification_tool(
|
||||
- You're about to perform a potentially dangerous operation
|
||||
- You have a recommendation but need user approval
|
||||
|
||||
Batching multiple independent questions:
|
||||
- **You CAN and SHOULD call ask_clarification multiple times in the same turn** when the user's
|
||||
request leaves several **independent** aspects unclear (e.g. project type AND tech stack AND
|
||||
deployment target). Emit one tool call per aspect — do NOT cram several unrelated questions
|
||||
into one `question` string.
|
||||
- The frontend automatically groups all clarification cards in one turn into a single submission,
|
||||
so the user answers them together in one reply. You will then receive one combined user message
|
||||
addressing every question in order.
|
||||
- When the questions DEPEND on each other (the second only makes sense after the first is answered),
|
||||
ask only the first one and wait for the reply before asking the next.
|
||||
|
||||
Best practices:
|
||||
- Ask ONE clarification at a time for clarity
|
||||
- Be specific and clear in your question
|
||||
- Don't make assumptions when clarification is needed
|
||||
- For risky operations, ALWAYS ask for confirmation
|
||||
- After calling this tool, execution will be interrupted automatically
|
||||
- One question per ask_clarification call — keep each `question` focused on a single decision.
|
||||
- Be specific and clear in your question.
|
||||
- Provide `options` (with concise labels) whenever there is a discrete set of reasonable choices —
|
||||
this lets the user multi-select in the UI instead of typing free text.
|
||||
- Don't make assumptions when clarification is needed.
|
||||
- For risky operations, ALWAYS ask for confirmation.
|
||||
- After calling this tool (or several in one turn), execution will be interrupted automatically.
|
||||
|
||||
Args:
|
||||
question: The clarification question to ask the user. Be specific and clear.
|
||||
question: The clarification question to ask the user. Focused on a single decision; be specific and clear.
|
||||
clarification_type: The type of clarification needed (missing_info, ambiguous_requirement, approach_choice, risk_confirmation, suggestion).
|
||||
context: Optional context explaining why clarification is needed. Helps the user understand the situation.
|
||||
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from.
|
||||
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from; the UI supports multi-select.
|
||||
"""
|
||||
# This is a placeholder implementation
|
||||
# The actual logic is handled by ClarificationMiddleware which intercepts this tool call
|
||||
|
||||
+13
-1
@@ -82,7 +82,19 @@ def memory_tool(
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
if not config.enabled or not config.builtin.enabled:
|
||||
if not config.enabled:
|
||||
return json.dumps(
|
||||
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
return json.dumps(
|
||||
{"success": False, "error": "当前使用 V1 记忆模式,记忆由系统自动提取,不支持手动写入。"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
if not config.builtin.enabled:
|
||||
return json.dumps(
|
||||
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
|
||||
ensure_ascii=False,
|
||||
|
||||
@@ -6,7 +6,7 @@ from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.reflection import resolve_variable
|
||||
from deerflow.sandbox.security import is_host_bash_allowed
|
||||
from deerflow.tools.builtins import ask_clarification_tool, memory_tool, present_file_tool, scheduled_task_tool, task_tool, view_image_tool
|
||||
from deerflow.tools.builtins import agent_orchestration_tool, ask_clarification_tool, memory_tool, present_file_tool, scheduled_task_tool, task_tool, view_image_tool
|
||||
from deerflow.tools.builtins.tool_search import reset_deferred_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
|
||||
BUILTIN_TOOLS = [
|
||||
present_file_tool,
|
||||
ask_clarification_tool,
|
||||
agent_orchestration_tool,
|
||||
]
|
||||
|
||||
SUBAGENT_TOOLS = [
|
||||
@@ -39,6 +40,7 @@ def get_available_tools(
|
||||
model_name: str | None = None,
|
||||
subagent_enabled: bool = False,
|
||||
is_scheduled_run: bool = False,
|
||||
excluded_tools: list[str] | None = None,
|
||||
*,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> list[BaseTool]:
|
||||
@@ -52,6 +54,9 @@ def get_available_tools(
|
||||
include_mcp: Whether to include tools from MCP servers (default: True).
|
||||
model_name: Optional model name to determine if vision tools should be included.
|
||||
subagent_enabled: Whether to include subagent tools (task, task_status).
|
||||
excluded_tools: Optional list of tool names to exclude from the result
|
||||
(applied AFTER all other resolution). Takes precedence over
|
||||
``app_config.excluded_tools``.
|
||||
|
||||
Returns:
|
||||
List of available tools.
|
||||
@@ -209,4 +214,19 @@ def get_available_tools(
|
||||
"Duplicate tool name %r detected and skipped — check your config.yaml and MCP server registrations (issue #1803).",
|
||||
t.name,
|
||||
)
|
||||
|
||||
# Drop tools the caller (or config) wants excluded. Request-level
|
||||
# ``excluded_tools`` wins; the config-level list is the static fallback.
|
||||
effective_excluded = excluded_tools if excluded_tools else getattr(config, "excluded_tools", None) or []
|
||||
if effective_excluded:
|
||||
excluded_set = set(effective_excluded)
|
||||
filtered_tools = [t for t in unique_tools if t.name not in excluded_set]
|
||||
if len(filtered_tools) < len(unique_tools):
|
||||
logger.info(
|
||||
"Excluded %d tool(s) by request/config: %s",
|
||||
len(unique_tools) - len(filtered_tools),
|
||||
effective_excluded,
|
||||
)
|
||||
unique_tools = filtered_tools
|
||||
|
||||
return unique_tools
|
||||
@@ -1,51 +1,53 @@
|
||||
[project]
|
||||
name = "deer-flow"
|
||||
version = "0.1.0"
|
||||
description = "LangGraph-based AI agent system with sandbox execution capabilities"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"deerflow-harness",
|
||||
"fastapi>=0.115.0",
|
||||
"httpx>=0.28.0",
|
||||
"python-multipart>=0.0.26",
|
||||
"sse-starlette>=2.1.0",
|
||||
"uvicorn[standard]>=0.34.0",
|
||||
"lark-oapi>=1.4.0",
|
||||
"slack-sdk>=3.33.0",
|
||||
"python-telegram-bot>=21.0",
|
||||
"langgraph-sdk>=0.1.51",
|
||||
"markdown-to-mrkdwn>=0.3.1",
|
||||
"wecom-aibot-python-sdk>=0.1.6",
|
||||
"dingtalk-stream>=0.24.3",
|
||||
"bcrypt>=4.0.0",
|
||||
"pyjwt>=2.9.0",
|
||||
"email-validator>=2.0.0",
|
||||
"croniter>=6.2.2",
|
||||
]
|
||||
|
||||
[project]
|
||||
name = "deer-flow"
|
||||
version = "0.1.0"
|
||||
description = "LangGraph-based AI agent system with sandbox execution capabilities"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"deerflow-harness",
|
||||
"fastapi>=0.115.0",
|
||||
"httpx>=0.28.0",
|
||||
"python-multipart>=0.0.26",
|
||||
"sse-starlette>=2.1.0",
|
||||
"uvicorn[standard]>=0.34.0",
|
||||
"lark-oapi>=1.4.0",
|
||||
"slack-sdk>=3.33.0",
|
||||
"python-telegram-bot>=21.0",
|
||||
"langgraph-sdk>=0.1.51",
|
||||
"markdown-to-mrkdwn>=0.3.1",
|
||||
"wecom-aibot-python-sdk>=0.1.6",
|
||||
"dingtalk-stream>=0.24.3",
|
||||
"bcrypt>=4.0.0",
|
||||
"pyjwt>=2.9.0",
|
||||
"email-validator>=2.0.0",
|
||||
"croniter>=6.2.2",
|
||||
"hindsight-client>=0.4.22",
|
||||
"asyncmy>=0.2.11",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
postgres = ["deerflow-harness[postgres]"]
|
||||
mysql = ["deerflow-harness[mysql]"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"prompt-toolkit>=3.0.0",
|
||||
"pytest>=9.0.3",
|
||||
"pytest-asyncio>=1.3.0",
|
||||
"ruff>=0.14.11",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
"no_auto_user: disable the conftest autouse contextvar fixture for this test",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
index-url = "https://pypi.org/simple"
|
||||
|
||||
[tool.uv.workspace]
|
||||
members = ["packages/harness"]
|
||||
|
||||
[tool.uv.sources]
|
||||
deerflow-harness = { workspace = true }
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"prompt-toolkit>=3.0.0",
|
||||
"pytest>=9.0.3",
|
||||
"pytest-asyncio>=1.3.0",
|
||||
"ruff>=0.14.11",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
"no_auto_user: disable the conftest autouse contextvar fixture for this test",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
index-url = "https://pypi.org/simple"
|
||||
|
||||
[tool.uv.workspace]
|
||||
members = ["packages/harness"]
|
||||
|
||||
[tool.uv.sources]
|
||||
deerflow-harness = { workspace = true }
|
||||
@@ -0,0 +1,179 @@
|
||||
"""AI 写作 assistant 冒烟脚本:验证 ai_writing graph 注册到 LangGraph Server 后,
|
||||
interrupt/Command(resume=...)/state 持久化/custom stream 全链路是否跑通。
|
||||
|
||||
阶段 0 用名为 ai_writing_probe;阶段 1 起改为正式 assistant 名 ai_writing,本脚本随之切换。
|
||||
|
||||
前置条件:
|
||||
1. langgraph.json 已注册 ai_writing(见 ai-writing-优化.md 阶段 1)。
|
||||
2. 后端已通过 `make dev` 启动,Gateway + LangGraph Server 监听 8001。
|
||||
3. config.yaml 配好至少一个可用 model,researcher 用到的 search_provider 也已配置
|
||||
—— 否则 researcher 节点会失败(这是业务依赖,不是迁移问题,仍然算「跑通」)。
|
||||
|
||||
运行:
|
||||
cd offline-backend-20260512/backend
|
||||
PYTHONPATH=. uv run python scripts/probe_ai_writing_assistant.py
|
||||
|
||||
通过条件(与 ai-writing-优化.md §阶段 0 §通过条件 对齐):
|
||||
- assistants 列表里能看到 ai_writing_probe
|
||||
- 跑到 pause_material_confirm 能拿到 interrupt 事件
|
||||
- Command(resume={"action": "confirm", ...}) 能成功续跑
|
||||
- 重建客户端后 client.threads.get_state(thread_id) 仍能拿到上一阶段 state
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from langgraph_sdk import get_client
|
||||
except ImportError:
|
||||
print("ERROR: langgraph-sdk 未安装。run: uv pip install langgraph-sdk", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
ASSISTANT_ID = "ai_writing"
|
||||
DEFAULT_URL = os.environ.get("DEERFLOW_LANGGRAPH_URL", "http://localhost:8001/api/langgraph")
|
||||
|
||||
|
||||
def _banner(text: str) -> None:
|
||||
print(f"\n{'=' * 8} {text} {'=' * 8}")
|
||||
|
||||
|
||||
def _summarize_event(event: Any) -> str:
|
||||
"""裁剪日志,避免一屏全是大段 markdown / 素材列表。"""
|
||||
s = repr(event)
|
||||
return s if len(s) <= 240 else s[:240] + "…"
|
||||
|
||||
|
||||
async def _stream_until_interrupt(
|
||||
client, thread_id: str, *, input_payload: dict | None = None, command: dict | None = None
|
||||
) -> dict | None:
|
||||
"""跑一次 runs.stream,捕获第一个 interrupt(含 pause payload)后返回,否则返回 None。
|
||||
|
||||
收到 interrupt 时把 payload 一并打出来,便于人工核对 4 个 pause 点的 schema。
|
||||
"""
|
||||
interrupt_payload: dict | None = None
|
||||
kwargs: dict[str, Any] = {
|
||||
"stream_mode": ["values", "messages-tuple", "custom", "updates"],
|
||||
}
|
||||
if input_payload is not None:
|
||||
kwargs["input"] = input_payload
|
||||
if command is not None:
|
||||
kwargs["command"] = command
|
||||
|
||||
async for chunk in client.runs.stream(thread_id, ASSISTANT_ID, **kwargs):
|
||||
# langgraph-sdk 把不同 mode 的事件混在同一 stream 里,按 chunk.event 分类。
|
||||
event_name = getattr(chunk, "event", None) or (
|
||||
chunk.get("event") if isinstance(chunk, dict) else None
|
||||
)
|
||||
data = getattr(chunk, "data", None) or (
|
||||
chunk.get("data") if isinstance(chunk, dict) else None
|
||||
)
|
||||
print(f"[stream] event={event_name} data={_summarize_event(data)}")
|
||||
|
||||
# SDK 在遇到 interrupt() 时会发一个特殊事件 —— 不同版本字段名不同,做下兼容。
|
||||
if event_name == "updates" and isinstance(data, dict):
|
||||
for _node, node_update in data.items():
|
||||
if isinstance(node_update, dict) and "__interrupt__" in node_update:
|
||||
interrupt_payload = node_update["__interrupt__"]
|
||||
if event_name in {"interrupt", "__interrupt__"}:
|
||||
interrupt_payload = data
|
||||
break
|
||||
|
||||
return interrupt_payload
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
print(f"LangGraph URL = {DEFAULT_URL}")
|
||||
client = get_client(url=DEFAULT_URL)
|
||||
|
||||
# ── Step 1: 确认 assistant 已注册 ──────────────────────────────────────────
|
||||
_banner("Step 1: 列 assistant,确认 probe 已注册")
|
||||
assistants = await client.assistants.search()
|
||||
names = [a.get("name") or a.get("graph_id") for a in assistants]
|
||||
print(f"已注册 assistants: {names}")
|
||||
if not any(ASSISTANT_ID in (n or "") for n in names):
|
||||
print(f"FAIL: 未找到 {ASSISTANT_ID},请检查 langgraph.json 注册是否生效。")
|
||||
return 1
|
||||
|
||||
# ── Step 2: 创建 thread 并触发第一次 run ──────────────────────────────────
|
||||
_banner("Step 2: 创建 thread + 启动写作(应停在 pause_material)")
|
||||
thread = await client.threads.create()
|
||||
thread_id = thread["thread_id"]
|
||||
print(f"thread_id = {thread_id}")
|
||||
|
||||
start_input = {
|
||||
"user_intent": "写一篇关于 LangGraph 多 worker 部署最佳实践的分析报道,目标读者是后端工程师,约 800 字",
|
||||
"article_type": "analysis",
|
||||
"target_audience": "后端工程师",
|
||||
"word_count_target": 800,
|
||||
"keyword_count": 4,
|
||||
"writing_mode": "loose",
|
||||
"max_revisions": 2,
|
||||
"status": "researching",
|
||||
"drafts": [],
|
||||
"progress_events": [],
|
||||
}
|
||||
payload = await _stream_until_interrupt(client, thread_id, input_payload=start_input)
|
||||
if not payload:
|
||||
print("FAIL: 未收到 interrupt 事件 —— pause_material 可能没停住,或 researcher 抛错。")
|
||||
return 2
|
||||
print(f"\nPAUSE-1 payload = {_summarize_event(payload)}")
|
||||
|
||||
# ── Step 3: 确认素材,应停在 pause_outline ────────────────────────────────
|
||||
_banner("Step 3: Command(resume) 确认素材(应停在 pause_outline)")
|
||||
payload = await _stream_until_interrupt(
|
||||
client, thread_id,
|
||||
command={"resume": {"action": "confirm"}}, # 不指定 approvedMaterialIds → 默认全部通过
|
||||
)
|
||||
if not payload:
|
||||
print("FAIL: 未停在 pause_outline。")
|
||||
return 3
|
||||
print(f"\nPAUSE-2 payload = {_summarize_event(payload)}")
|
||||
|
||||
# ── Step 4: 确认大纲,应停在 pause_draft 或 pause_section_help ───────────
|
||||
_banner("Step 4: Command(resume) 确认大纲(应停在 pause_draft 或 pause_section_help)")
|
||||
payload = await _stream_until_interrupt(
|
||||
client, thread_id,
|
||||
command={"resume": {"action": "confirm"}},
|
||||
)
|
||||
if not payload:
|
||||
print("FAIL: 未停在 pause_draft / pause_section_help。")
|
||||
return 4
|
||||
print(f"\nPAUSE-3 payload = {_summarize_event(payload)}")
|
||||
|
||||
# ── Step 5: 定稿 ─────────────────────────────────────────────────────────
|
||||
_banner("Step 5: Command(resume) finalize 草稿(应跑到 END)")
|
||||
payload = await _stream_until_interrupt(
|
||||
client, thread_id,
|
||||
command={"resume": {"action": "finalize"}},
|
||||
)
|
||||
if payload:
|
||||
print(f"NOTE: 收到额外 interrupt(可能编辑审核打回) payload = {_summarize_event(payload)}")
|
||||
print("尝试强制定稿…")
|
||||
payload = await _stream_until_interrupt(
|
||||
client, thread_id,
|
||||
command={"resume": {"action": "force_finalize", "overrideVerdict": True}},
|
||||
)
|
||||
if payload:
|
||||
print("WARN: 强制定稿后仍有 interrupt,graph 路由可能有意外分支。")
|
||||
|
||||
# ── Step 6: 验证 state 持久化 ────────────────────────────────────────────
|
||||
_banner("Step 6: 重新建客户端 client.threads.get_state(thread_id) 应能拿到最终 state")
|
||||
client2 = get_client(url=DEFAULT_URL)
|
||||
state = await client2.threads.get_state(thread_id)
|
||||
values = state.get("values") or {}
|
||||
final_status = values.get("status")
|
||||
has_draft = bool(values.get("current_draft"))
|
||||
print(f"final status = {final_status}, has_draft = {has_draft}, "
|
||||
f"next nodes = {state.get('next')}")
|
||||
if final_status not in {"done", "writing_draft", "revising"}:
|
||||
print("WARN: 最终 status 不是 done,请人工确认上面流程是否正常。")
|
||||
print("\nPROBE OK —— 阶段 0 验证通过的核心条件已成立(interrupt + resume + state 持久化)。")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Tests for the ``POST /api/skills/reconcile-custom`` adoption flow.
|
||||
|
||||
Covers the scenario where a user drops a skill directory onto disk by hand
|
||||
(bypassing the upload API) and then calls the reconcile route so it shows up
|
||||
in the UI.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway.routers import skills as skills_router
|
||||
from deerflow.persistence.skills import MemorySkillStore
|
||||
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
|
||||
|
||||
|
||||
def _write_skill_on_disk(root: Path, name: str, description: str = "from disk") -> None:
|
||||
"""Drop a valid SKILL.md into ``<root>/custom/<name>/`` like a manual unzip would."""
|
||||
skill_dir = root / "custom" / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
f"---\nname: {name}\ndescription: {description}\n---\n# {name}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _fake_request(user_id: str | None = "user-a") -> SimpleNamespace:
|
||||
user = SimpleNamespace(id=user_id, system_role="user") if user_id else None
|
||||
return SimpleNamespace(state=SimpleNamespace(user=user))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skills_root(tmp_path: Path) -> Path:
|
||||
root = tmp_path / "skills"
|
||||
(root / "public").mkdir(parents=True)
|
||||
(root / "custom").mkdir(parents=True)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_storage(
|
||||
skills_root: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> LocalSkillStorage:
|
||||
"""Force the router to use a LocalSkillStorage rooted at ``skills_root``."""
|
||||
storage = LocalSkillStorage(host_path=str(skills_root))
|
||||
monkeypatch.setattr(
|
||||
skills_router, "get_or_new_skill_storage", lambda **_: storage
|
||||
)
|
||||
|
||||
async def _noop_refresh() -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
skills_router, "refresh_skills_system_prompt_cache_async", _noop_refresh
|
||||
)
|
||||
return storage
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_adopts_orphan_custom_skill(
|
||||
skills_root: Path, patched_storage: LocalSkillStorage
|
||||
):
|
||||
_write_skill_on_disk(skills_root, "manually-dropped")
|
||||
store = MemorySkillStore()
|
||||
|
||||
result = await skills_router.reconcile_custom_skills(
|
||||
request=_fake_request("user-a"), config=None, skill_store=store
|
||||
)
|
||||
|
||||
assert result.adopted == ["manually-dropped"]
|
||||
assert result.already_registered == []
|
||||
|
||||
record = await store.get_any("manually-dropped")
|
||||
assert record is not None
|
||||
assert record["owner_user_id"] == "user-a"
|
||||
assert record["published"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_skips_already_registered(
|
||||
skills_root: Path, patched_storage: LocalSkillStorage
|
||||
):
|
||||
_write_skill_on_disk(skills_root, "already-known")
|
||||
store = MemorySkillStore()
|
||||
# Pre-existing ownership row — should be left alone.
|
||||
await store.create(
|
||||
{"name": "already-known", "owner_user_id": "user-b", "published": True}
|
||||
)
|
||||
|
||||
result = await skills_router.reconcile_custom_skills(
|
||||
request=_fake_request("user-a"), config=None, skill_store=store
|
||||
)
|
||||
|
||||
assert result.adopted == []
|
||||
assert result.already_registered == ["already-known"]
|
||||
|
||||
# Ownership must not be overwritten.
|
||||
record = await store.get_any("already-known")
|
||||
assert record is not None
|
||||
assert record["owner_user_id"] == "user-b"
|
||||
assert record["published"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_mixed_adopts_only_orphans(
|
||||
skills_root: Path, patched_storage: LocalSkillStorage
|
||||
):
|
||||
_write_skill_on_disk(skills_root, "orphan-one")
|
||||
_write_skill_on_disk(skills_root, "orphan-two")
|
||||
_write_skill_on_disk(skills_root, "registered")
|
||||
store = MemorySkillStore()
|
||||
await store.create(
|
||||
{"name": "registered", "owner_user_id": "user-a", "published": False}
|
||||
)
|
||||
|
||||
result = await skills_router.reconcile_custom_skills(
|
||||
request=_fake_request("user-a"), config=None, skill_store=store
|
||||
)
|
||||
|
||||
assert sorted(result.adopted) == ["orphan-one", "orphan-two"]
|
||||
assert result.already_registered == ["registered"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_with_empty_custom_dir_is_noop(
|
||||
skills_root: Path, patched_storage: LocalSkillStorage
|
||||
):
|
||||
store = MemorySkillStore()
|
||||
|
||||
result = await skills_router.reconcile_custom_skills(
|
||||
request=_fake_request("user-a"), config=None, skill_store=store
|
||||
)
|
||||
|
||||
assert result.adopted == []
|
||||
assert result.already_registered == []
|
||||
Generated
+2706
-2699
File diff suppressed because it is too large.
Load diff
@@ -37,6 +37,26 @@ models:
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: minimax-m2.7
|
||||
display_name: MiniMax M2.7
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.7
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: minimax-m2.5
|
||||
display_name: MiniMax M2.5
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.5
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
tool_groups:
|
||||
- name: web
|
||||
- name: file:read
|
||||
@@ -89,6 +109,8 @@ title:
|
||||
Assistant: {assistant_msg}
|
||||
skills:
|
||||
path: /app/skills
|
||||
security_scan:
|
||||
enabled: false
|
||||
agents_api:
|
||||
enabled: true
|
||||
database:
|
||||
@@ -97,13 +119,28 @@ database:
|
||||
pool_size: 10
|
||||
echo_sql: false
|
||||
checkpointer:
|
||||
# SQLite 文件路径用相对路径,由 resolve_sqlite_conn_str 解析成绝对路径。
|
||||
# Docker 部署:把容器内 .deer-flow 目录挂卷出来即可持久化(撑住容器重启)。
|
||||
# 这里同时被「主 agent」和「AI 写作 graph」共用 —— LangGraph 用
|
||||
# (thread_id, checkpoint_ns) 做隔离,session_id 是 uuid,不会撞。
|
||||
type: sqlite
|
||||
connection_string: D:\cursor-work\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
|
||||
connection_string: .deer-flow/data/checkpoints.db
|
||||
# AI 写作子系统的会话清理配置。
|
||||
# 业务表(ai_writing_sessions @ MySQL) + checkpointer(checkpoints.db @ SQLite) 联动清理。
|
||||
# 留意:清理走 cron 表达式调度,默认每天凌晨 3 点跑一次,按 updated_at 一刷拉清。
|
||||
ai_writing:
|
||||
cleanup:
|
||||
enabled: true
|
||||
retention_days: 7 # 保留 7 天以内的会话,更早的整条删
|
||||
cron: "0 3 * * *" # 每天 03:00 本地时区
|
||||
delete_only_finished: false # false=不看状态一刷拉;true=只清 done/error
|
||||
batch_size: 500 # 单批最多删多少行,避免长事务
|
||||
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
|
||||
# 三层覆盖优先级(高->低):
|
||||
# users/{user_id}/memory_config.json > .deer-flow/memory_user_overrides.json > 此处
|
||||
memory:
|
||||
enabled: true
|
||||
version: v1
|
||||
provider: hindsight # 'builtin' | 'hindsight'(外部 provider 名)
|
||||
# builtin 始终启用,这里只决定 external 选哪个
|
||||
builtin:
|
||||
@@ -114,7 +151,7 @@ memory:
|
||||
|
||||
hindsight: # provider=hindsight 时生效,需另行部署 Hindsight 服务
|
||||
mode: local_external # 'cloud' | 'local_embedded' | 'local_external'
|
||||
api_url: http://localhost:8765
|
||||
api_url: http://47.88.25.99:18888
|
||||
api_key: "" # 部署 Hindsight 后填,或改成 $HINDSIGHT_API_KEY 并设同名环境变量
|
||||
# 注意:写成 $VAR 但环境变量未设置会导致配置加载失败
|
||||
bank_id_template: "deerflow-user-{user_id}"
|
||||
|
||||
Reference in new issue
Block a user