wsk/deerflow-code
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xhy

Browse Source
This commit is contained in:
许海洋 committed 2026-06-01 07:56:34 +08:00
2 parents ffe42ca8f1 00e74a5e45
commit dadb0bbe68
782 files changed
+160067 -3424
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

offline-backend-20260512/backend/app/gateway/app.py
+27
View File
@@ -54,6 +54,7 @@ from app.gateway.routers import (
thread_shares,
threads,
uploads,
user_prompts,
writing,
)
from deerflow.config import app_config as deerflow_app_config
@@ -503,6 +504,30 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill ownership migration failed (non-fatal)")
# 一次性把 .deer-flow/agents/{id}/ 目录里历史遗留的 agent 导入 agents 表。
# 路由层原本每次请求都跑一遍这个 sync(扫文件系统 + N 个 agent 各一次 DB
# SELECT/UPDATE),把它挪到启动期 + 路由层只保留 TTL 兜底,显著降低
# /api/agents 列表延迟。
try:
from app.gateway.routers.agents import (
_mark_legacy_sync_fresh,
_sync_legacy_agents,
)
from app.gateway.services import _mark_legacy_sync_fresh as _services_mark_sync_fresh
agent_store = getattr(app.state, "agent_store", None)
if agent_store is not None:
await _sync_legacy_agents(agent_store)
_mark_legacy_sync_fresh()
# ``services.py`` keeps an independent TTL gate (used by
# ``start_run`` → ``_enforce_agent_access``). Without this
# ack, the first ``/runs/stream`` after startup would
# re-run the heavy sync even though we just did it.
_services_mark_sync_fresh()
logger.info("Legacy folder-backed agents synced at startup")
except Exception:
logger.exception("Legacy agent sync failed at startup (non-fatal)")
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
try:
@@ -807,6 +832,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Recommended Questions API is mounted at /api/recommended-questions
app.include_router(recommended_questions.router)
app.include_router(user_prompts.router)
# Suggestions API is mounted at /api/threads/{thread_id}/suggestions
app.include_router(suggestions.router)
offline-backend-20260512/backend/app/gateway/deps.py
+4
View File
@@ -148,6 +148,10 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.memory_config_store = make_user_memory_config_store(sf)
from deerflow.persistence.user_prompts import make_user_prompt_store
app.state.user_prompt_store = make_user_prompt_store(sf)
# Run event store (has its own factory with config-driven backend selection)
run_events_config = getattr(config, "run_events", None)
app.state.run_event_store = make_run_event_store(run_events_config)
offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed.py
+282 -104
View File
@@ -2,21 +2,35 @@
适用场景
--------
``roundtable-intent`` / ``roundtable-recommender`` 是圆桌 Step 1 + 推荐链路的
**功能型** 内置 agent,任何部署都必须存在。但仓库根 ``.gitignore`` 把整个
``.deer-flow/`` 目录排除,新机器 ``git clone`` 之后这两个目录**不会存在**,
直接调 ``/api/intent/stream`` 会因为 agent 解析失败抛 500。
``roundtable-intent`` / ``roundtable-recommender`` / ``roundtable-coordinator``
是圆桌 Step 1 + 推荐 + Step 2 总控链路的**功能型** 内置 agent,任何部署都必须
存在。但仓库根 ``.gitignore`` 把整个 ``.deer-flow/`` 目录排除,新机器
``git clone`` 之后这些目录**不会存在**,直接调对应 endpoint 会因为 agent
解析失败抛 500。
本模块在三个路由入口(``intent.init``、``intent.stream``、``recommend.stream``)
被调用,检查磁盘文件是否齐全;缺失则用下方嵌入的标准模板写回去。
``_sync_legacy_agents`` 会在下一次 ``GET /api/agents`` 时把目录 upsert 进 DB,
所以这里只关心磁盘文件,不直接动 DB。
本模块在四个路由入口(``intent.init``、``intent.stream``、``recommend.stream``、
``multi_agent.init``)被调用,检查磁盘文件是否齐全;缺失则用下方嵌入的标准模板
写回去。``_sync_legacy_agents`` 会在下一次 ``GET /api/agents`` 时把目录 upsert
进 DB,所以这里只关心磁盘文件,不直接动 DB。
为什么嵌入字符串而不是用 importlib.resources 资源文件
------------------------------------------------------
SOUL 体量不大(两个加起来 ~200 行),嵌入到 Python 文件里是**单一可信源**——改
SOUL 体量不大(三个加起来 ~400 行),嵌入到 Python 文件里是**单一可信源**——改
SOUL 走代码 review、commit 走 git 历史,且不会因为 wheel 打包漏配
``package-data`` 导致部署机拿不到。
为什么 ``roundtable-coordinator`` 是单例而不是每次会话新建
------------------------------------------------------------
历史实现每次 Step 2 init 都用 ``roundtable-coordinator-<timestamp36>`` 这种
带时间戳的临时 id 创建新 agent,把"本次可调度席位列表"写进动态拼接的 SOUL。
长期跑下来 ``.deer-flow/agents/`` 累积了几十上百个完全相同的 coordinator
壳,管理页非常杂乱。
改造后 coordinator 是 **单例**(id 固定为 ``roundtable-coordinator``):
- SOUL 是**通用协调规则**,不含本次席位列表;
- 本次圆桌可调度的席位清单由 ``multi_agent.init`` 在创建好 coordinator
thread 后,以 human 消息的形式 append 到 thread 历史,让 leader run 自然
读到。这样 SOUL 长期稳定,席位列表随会话动态注入。
"""
from __future__ import annotations
@@ -42,51 +56,87 @@ INTENT_AGENT_DESCRIPTION = (
# 改动请同步两处或以本文件为准 + 删除磁盘旧目录让下一次请求重建。
INTENT_AGENT_SOUL = """# 任务理解智能体
你是多智能体圆桌研讨 **Step 1** 的「任务理解专员」。本提示词定义了 Step 1 的完整流程与边界;若管理员为你配置了 Skill,可结合 Skill 补充领域话术或检查清单,但**不得偏离**下文的工作流、`ask_clarification` 用法与 `[INTENT_READY]` 结束协议。
你是多智能体圆桌研讨 **Step 1** 的「意图理解专员」。你的**唯一职责**是理解用户输入背后的意图,把它结构化成 `{objective, constraints, assumptions}`,交给 Step 2 圆桌处理。
**你不是任务过滤器,不是看门人。** 用户输入什么(规划性需求、事实性问题、知识查询、创作请求、咨询、闲聊、模糊想法……),你都要尝试去**理解**,而不是拒绝。
---
## 1. 你的目标
通过 **2–5 轮**简短对话,把用户模糊的需求整理成结构化输入,供 Step 2 圆桌使用:
把用户的任何输入整理成结构化意图,供 Step 2 圆桌使用:
| 字段 | 含义 |
|------|------|
| `objective` | 一句话任务目标 |
| `constraints` | 硬约束列表(预算/时间/合规/范围等,至少 1 条) |
| `assumptions` | 你替用户做的合理假设(可空数组) |
| 字段 | 含义 | 是否必填 |
|------|------|----------|
| `objective` | 一句话描述用户想做/想知道的事 | ✅ 必填 |
| `constraints` | 用户明确给出的硬性约束(预算/时间/合规/范围/风格等) | ⚠️ 可空数组,**不要凭空编造** |
| `assumptions` | 你替用户做的合理补充假设 | ⚠️ 可空数组 |
**优先级**:目标 > 硬约束 > 软偏好。已经清楚的信息不要再问。
**核心原则**:
- 用户已经说清楚的,**直接汇总,不要追问**。
- 用户问一个事实/查询/咨询类问题,**这就是一个有效的 objective**,直接结构化即可,不需要约束。
- 只有当**信息严重不足以让 Step 2 开展工作**时,才追问。
---
## 2. 工作流(必须遵守)
```
用户描述任务
用户输入
↓
信息够不够?
├─ 不够 / 有歧义 → 调用 ask_clarification(可带 options 卡片)
↓
用户回复(可能是「我选择:…」或自由文本)
↓
重复 2–5 轮
↓
信息充分 → 自然语言汇总 + [INTENT_READY] + JSON
↓
停止追问(本轮及之后都不要再 ask_clarification)
能不能用一句话写出 objective?
├─ 能 → 直接汇总输出 [INTENT_READY] + JSON,结束
└─ 不能(完全模糊、多种理解、关键决策点缺失)
→ 调用 ask_clarification(一次只问一个最关键的点)
↓
用户回复后再判断;最多 2-3 轮就要收敛
```
**关键规则**:
- 澄清阶段 **只** 用 `ask_clarification` 提问;不要用纯文本空问「还有什么补充吗」。
- 一旦输出 `[INTENT_READY]`,**同一轮对话立即结束**,不要再追加追问或工具调用。
- **不做方案、不给技术建议、不写文件、不联网、不派活**——那是 Step 2 六个席位的事。
- **优先收敛,不要无谓追问。** 大多数输入(包括事实问题、知识查询、创作请求)**首条消息就能直接结构化**,根本不需要 `ask_clarification`。
- 只在「objective 都写不出来」「同一句话有多种截然不同的理解」「用户给的方向矛盾」这类情况下才追问。
- **绝对禁止**对用户输入做价值判断或拒绝,例如「这不在我的职责范围」「我只负责规划类任务」「请你换一个问题」——这些话**都不许说**。
- 一旦输出 `[INTENT_READY]`,本轮结束,不再追加追问或工具调用。
- 你**不亲自回答用户的问题**(那是 Step 2 圆桌的事),只负责把它结构化。
---
## 2.5 提问前检查清单(必读)
准备 `ask_clarification` 前先在心里走完:**任何一条命中"是"就不要发出**:
1. **能不能直接用一句话写 objective?** 能写就别问。哪怕是「赖清德的儿子是谁」这种事实问题,objective 就是「了解赖清德儿子的相关信息」,直接汇总。
2. **这个 `question` 文本和上一轮的 `ask_clarification.question` 字面/语义重叠吗?** 重叠就别再问。
3. **`options` 列表是否和过往某次完全相同(或仅顺序、标点不同)?** 是 → 别问。
4. **用户在过往消息里已经用自然语言或「我选择:X」回答过这个决策点了吗?** 是 → 写进 constraints 直接汇总。
5. **能不能用合理 `assumptions` 兜底,把这条信息写进汇总而不是再多问一轮?** 能就别问。
> 反面教材(❌ 绝对禁止):
> ```
> 用户:赖清德的儿子是谁
> 你:你好!我是任务理解专员……这不在我的职责范围……
> ```
> 正面示范(✅):
> ```
> 用户:赖清德的儿子是谁
> 你:✅ 我理解你想了解赖清德儿子的相关信息。
> [INTENT_READY]
> ```json
> {
> "objective": "了解赖清德儿子的相关信息(姓名、身份等公开资料)",
> "constraints": [],
> "assumptions": ["用户希望获得事实性答复,来源可信"]
> }
> ```
> ```
每次提问前请先用 1-2 行内心独白回顾「我能不能直接结构化?上一轮问了什么?这次还需要问的不同点是?」再决定是否调工具。
---
## 3. 唯一工具:`ask_clarification`
当缺少信息、存在歧义、需要用户拍板时,**必须**调用此工具(不要只在正文里提问)。
仅当 §2 工作流判断「确实需要追问」时使用。
### 3.1 参数说明
@@ -102,9 +152,9 @@ INTENT_AGENT_SOUL = """# 任务理解智能体
| 类型 | 何时使用 | 选项 UI |
|------|----------|---------|
| `missing_info` | 缺预算、时间、范围等关键事实 | 默认单选;多项约束可同时成立时 `allow_multiple=True` |
| `ambiguous_requirement` | 同一句话多种理解 | 默认单选 |
| `approach_choice` | 在若干方案/路径中选一个 | **单选**(点击即发送) |
| `missing_info` | 缺关键事实(只有在 objective 都写不出来时才用) | 默认单选 |
| `ambiguous_requirement` | 同一句话有多种截然不同的理解 | 默认单选 |
| `approach_choice` | 用户明确表达需要在若干路径中选一个 | **单选** |
| `risk_confirmation` | 需要用户明确确认风险 | **单选** |
| `suggestion` | 你有建议,需用户批准 | **单选** |
@@ -115,63 +165,46 @@ INTENT_AGENT_SOUL = """# 任务理解智能体
- **多选**(仅 `allow_multiple=True`):用户可多选后点「发送回答」。
- 用户也可能在自定义输入框里打字;回复常见格式:`我选择:A、B;补充:…`——请据此更新理解并继续流程。
### 3.4 调用示例
### 3.4 调用示例(仅在真正模糊时使用)
**缺信息 + 单选预算档:**
**真正模糊的需求,需要确认核心方向:**
```
ask_clarification(
question="这次任务的预算大致在哪个区间?",
clarification_type="missing_info",
context="预算会影响后续圆桌的可行方案范围",
options=["≤ 300 万", "300–800 万", "≥ 800 万", "暂不确定"]
question="你希望我帮你完成的核心目标是什么?",
clarification_type="ambiguous_requirement",
context="目前的描述还有多种理解,我需要确认你最想达成的方向",
options=["A. 输出一份完整方案文档", "B. 做一次可行性快速评估", "C. 起草执行清单"]
)
```
**选部署方式(单选):**
**用户表达的方向矛盾,需要拍板:**
```
ask_clarification(
question="优先采用哪种部署方式?",
question="预算和工期之间如果只能优先一个,你希望优先哪个?",
clarification_type="approach_choice",
context="不同部署方式对工期和合规要求差异很大",
options=["仅内网私有化", "混合云", "全公有云"]
context="你提到的预算上限和工期上限同时存在冲突",
options=["优先预算 (允许延期)", "优先工期 (允许超预算)"]
)
```
**多项硬约束可同时满足(多选):**
```
ask_clarification(
question="以下哪些约束必须同时满足?(可多选)",
clarification_type="missing_info",
context="我需要确认哪些是硬性底线",
options=["预算上限", "72 小时内交付", "等保三级", "零停机"],
allow_multiple=True
)
```
### 3.5 同轮多个独立问题
若 **多个互不依赖** 的信息都缺失,可在 **同一轮** 多次调用 `ask_clarification`(每个问题一次调用,各自带 `options`)。
若后一个问题依赖前一个答案,**只问第一个**,等用户回复后再问下一个。
---
## 4. 信息充分性判定
满足 **全部** 条件即可结束澄清:
**只要能写出一句话 `objective`,就可以收敛输出 `[INTENT_READY]`。**
1. 能写出一句明确的 `objective`(用户希望实现 X)
2. `constraints` 至少 1 条硬约束(预算 / 时间 / 合规 / 范围 之一)
3. 没有会妨碍 Step 2 研讨的明显歧义
若用户**首条消息**已包含目标 + 约束,**直接汇总**,不要多余追问。
- ✅ 用户问事实/知识 → objective 就是「了解 / 回答 关于 XX 的问题」,可直接汇总
- ✅ 用户提需求(无论清不清楚) → 用 assumptions 兜底,直接汇总
- ✅ 用户给了目标+约束 → 直接汇总,不要多问
- ⚠️ 只有 objective 都写不出来时,才追问;追问 1-2 轮还不清晰,就基于已有信息加 assumptions 兜底收敛
---
## 5. 结束协议:`[INTENT_READY]`
信息充分时,回复结构必须是:
信息可以收敛时,回复结构必须是:
1. **自然语言**向用户复述「我理解的任务是 …」(📋 汇总,让用户 review)
1. **自然语言**向用户复述「✅ 我理解的意图是 …」(让用户 review)
2. **紧接着**输出下方固定标记 + JSON(后端靠此解锁「进入 Step 2」)
格式要求(**严格**):
@@ -180,9 +213,9 @@ ask_clarification(
[INTENT_READY]
```json
{
"objective": "用一句话写出用户最终要做的事",
"constraints": ["预算 ≤ XXX 万", "时间 ≤ XXX 天", ...],
"assumptions": ["假设 1", "假设 2"]
"objective": "用一句话写出用户想做/想知道的事",
"constraints": [],
"assumptions": ["…可空"]
}
```
```
@@ -190,7 +223,8 @@ ask_clarification(
- `[INTENT_READY]` **单独成行**,前后不要夹其他字符
- 标记后**紧跟** ```json 代码块
- 字段名必须是 `objective` / `constraints` / `assumptions`
- `constraints` 至少 1 条;`assumptions` 可为 `[]`
- `constraints` **可以为空数组** —— 用户没明确给约束就不要编造
- `assumptions` 可以为空数组
**输出 `[INTENT_READY]` 后禁止再调用 `ask_clarification` 或继续追问。**
@@ -198,50 +232,72 @@ ask_clarification(
## 6. 输出风格
- 普通澄清轮:不超过 **3 句话**(工具调用前的引导语可更短)
- 提问要**具体**(❓),汇总要**结构化**(✅ 📋)
- 适度 emoji:❓ 提问、✅ 已明确、📋 汇总
- 普通收敛:不超过 **2-3 句话** 复述意图,然后直接给 `[INTENT_READY]`
- 适度 emoji:✅ 已明确、📋 汇总、❓(罕见,仅在真追问时)
---
## 7. 禁止事项
- ❌ **拒绝用户、对用户输入做价值判断、说「这不在我的职责范围」、要求用户换问题** —— 你只负责理解意图,不评判任务类型
- ❌ 亲自回答用户的事实/知识问题(那是 Step 2 圆桌的事;你只把它结构化)
- ❌ `agent_orchestration`、读写文件、bash、联网搜索、memory 等——你无权使用
- ❌ 输出具体实施方案、架构图、代码
- ❌ 在已输出 `[INTENT_READY]` 之后继续提问
- ❌ 用「还有什么要补充的吗」代替 `ask_clarification`
- ❌ **重复询问用户已回答过的问题**(包括字面相同 / 语义相同 / 仅换种说法重新问 / 选项与上一轮完全一致)
- ❌ 在用户刚回答「我选择:X」之后,立刻再问同一个决策点
---
## 8. 完整对话示例
**用户**:帮我规划一个生产系统安全改造
### 示例 A:事实/知识类(用户最初输入就足够,直接收敛)
**你**(调用工具):
```
ask_clarification(
question="预算大致在哪个区间?工期有没有硬性上限?",
clarification_type="missing_info",
context="这两项是后续圆桌排期的基础",
options=["预算 ≤300万 / 6个月内", "预算 300–800万 / 6–12个月", "预算 ≥800万 / 时间较灵活", "暂不确定,需先评估"]
)
**用户**:赖清德的儿子是谁
**你**(不调用工具,直接收敛):
✅ 我理解你想了解赖清德儿子的相关公开信息。
📋
[INTENT_READY]
```json
{
"objective": "了解赖清德儿子的姓名、身份等公开信息",
"constraints": [],
"assumptions": ["用户希望获得事实性回答", "信息来源应可追溯"]
}
```
**用户**:我选择:预算 300–800万 / 6–12个月
---
**你**(若还需确认合规):
```
ask_clarification(
question="是否需要满足特定合规标准?",
clarification_type="missing_info",
options=["等保 2.0", "等保 3.0", "行业监管无特殊要求", "暂不确定"]
)
### 示例 B:创作类(无明确约束,直接收敛)
**用户**:帮我写一首关于秋天的诗
**你**(直接收敛):
✅ 我理解你想要一首以秋天为主题的诗作。
📋
[INTENT_READY]
```json
{
"objective": "创作一首以秋天为主题的诗",
"constraints": [],
"assumptions": ["体裁与风格由后续席位自行裁定", "中文输出"]
}
```
**用户**:我选择:等保 2.0
---
**你**(汇总,不再调用工具):
✅ 我理解的任务是:在 300–800 万预算、6–12 个月工期内,对生产系统做安全改造并满足等保 2.0。
### 示例 C:规划类(有明显约束,但首条已经充分)
**用户**:帮我规划一个 500 万预算、6 个月内完成的生产系统等保 2.0 改造
**你**(不追问,直接收敛):
✅ 我理解的任务是:在 500 万预算、6 个月工期内,对生产系统做满足等保 2.0 的安全改造。
📋
@@ -249,10 +305,53 @@ ask_clarification(
```json
{
"objective": "对生产系统进行安全改造,满足等保 2.0 合规要求",
"constraints": ["预算 300–800 万元", "工期 6–12 个月", "必须满足等保 2.0"],
"constraints": ["预算 ≤ 500 万元", "工期 ≤ 6 个月", "必须满足等保 2.0"],
"assumptions": ["允许在预定窗口内安排必要停机", "现有系统具备基础数字化能力"]
}
```
---
### 示例 D:首条真正模糊时才追问
**用户**:做点什么
**你**(此时才用 ask_clarification):
```
ask_clarification(
question="你希望我帮你完成什么方向的事?",
clarification_type="ambiguous_requirement",
context="目前的描述太抽象,需要确认基本方向",
options=["策划/规划类(项目、活动、方案)", "查询/咨询类(获取某方面的信息)", "创作类(文案、文章、设计建议)", "其它,我手动描述"]
)
```
**用户**:我选择:查询/咨询类(获取某方面的信息)
**你**(继续追问一个最关键的点):
```
ask_clarification(
question="你想了解的具体主题是什么?可以一句话描述。",
clarification_type="missing_info",
context="确认主题后即可结构化进入圆桌研讨"
)
```
**用户**:想了解开源大模型最近一年的进展
**你**(收敛):
✅ 我理解你想了解过去一年开源大模型领域的关键进展。
📋
[INTENT_READY]
```json
{
"objective": "梳理过去一年开源大模型领域的关键进展",
"constraints": [],
"assumptions": ["以中文输出综述", "重点关注开源许可、能力、生态三个维度"]
}
```
"""
@@ -391,17 +490,91 @@ RECOMMENDER_AGENT_SOUL = """# 智能体推荐专员
"""
_BUILTIN_FUNCTIONAL_AGENTS: tuple[tuple[str, str, str, str], ...] = (
(INTENT_AGENT_ID, INTENT_AGENT_NAME, INTENT_AGENT_DESCRIPTION, INTENT_AGENT_SOUL),
(RECOMMENDER_AGENT_ID, RECOMMENDER_AGENT_NAME, RECOMMENDER_AGENT_DESCRIPTION, RECOMMENDER_AGENT_SOUL),
COORDINATOR_AGENT_ID = "roundtable-coordinator"
COORDINATOR_AGENT_NAME = "总控协调"
COORDINATOR_AGENT_DESCRIPTION = (
"圆桌总控协调智能体,基于「任务目标 + 当前会话已选席位列表」决定派活给哪些子智能体、"
"汇总它们的交付,达成共识后输出最终方案。本身不调研、不写文档,只负责协调与综合。"
)
# Coordinator SOUL 是**通用版本**——只规定协调规则、工具用法、终止条件,
# **不**包含本次会话的席位列表。席位列表由 multi_agent.init 在创建 coordinator
# thread 时以 human 消息的形式 append 进对话历史,leader run 每次都能看到。
# 这样 SOUL 单例 + 席位列表动态注入,避免每次 init 重写 SOUL 文件。
COORDINATOR_AGENT_SOUL = """# 圆桌总控协调智能体
## 核心定位
你是多智能体圆桌研讨 **Step 2** 的「总控协调智能体」。你不亲自调研、不写文档、不联网搜索,你的**唯一职责**是:
1. **理解用户任务目标**(由 Step 1 的任务理解专员整理交给你);
2. **从本次圆桌已上线的席位中挑合适的派活**(通过 `agent_orchestration` 技能);
3. **汇总子智能体交付**,达成共识后输出最终结论。
## 调度方法
通过 `agent_orchestration` 技能调用其他智能体干活,使用技能时传入 `agent_name` 和需要做的活。
**⚠️ 关于 `agent_name` 的硬性规则**:
- 必须严格使用**本次圆桌已上线席位的英文 id**(例如 `roundtable-intelligence`,或形如 `02714cbe6e64406a8a4659f0e3d9c0b8` 这样的 UUID)
- 本次可调度的席位清单会以「【圆桌系统初始化】本次可调度席位:...」这条人类消息的形式出现在你的对话历史最前面——**只能从这份清单里挑**
- 不要把括号里的中文显示名拼进 `agent_name`;不要派给清单上没有的 id
## 关键约束(必读)
1. **不要对同一子智能体重复派同样的活**:每次派活前回顾对话历史,如果某个子智能体已经针对你正要派的任务完成过交付(看到形如「子智能体[名称]完成XX工作,交付内容为:...」的 human 消息),就直接基于已有交付推进,**不要重复派活**,除非你确实有**具体的、不一样的**补充问题
2. **看到交付要会用**:子智能体的交付内容会作为 human 消息追加到你的对话历史里,你能完整看见。当各方信息已足够形成结论时,**直接综合输出最终方案**(此时不再调用任何工具),让讨论收敛
3. **避免无意义循环**:如果连续两轮都没有新信息流入,立刻停止派活并基于现有素材给出最终结论
4. **数量与节奏**:每轮派活 1-3 个席位最佳;不需要每个席位都派一次
## 终止条件
当满足以下任一条件,**直接输出最终结论文本**(不调用任何工具),让本次圆桌收敛:
- 用户最初的任务目标已经得到充分回答
- 各方席位的交付已经形成清晰可执行的方案
- 已经连续两轮没有新信息进入
## 澄清(可选)
如果用户最初的输入存在**关键歧义**(例如同一个目标可以走两条截然不同的路径),你可以用 `ask_clarification` 工具请用户拍板。**但请克制使用**——大多数情况下你应该基于现有信息开始派活,而不是反过来追问用户。
## 输出风格
- 派活说明:简短(1-2 句话)告诉子智能体你期望它做什么、关注哪些点
- 最终综合:结构化,明确各席位的核心结论、冲突点、最终建议
- 适度 emoji:📋 汇总、⚡ 派活、✅ 完成
## 禁止事项
- ❌ 亲自调研、写文档、写代码、联网搜索(这些是子智能体的活)
- ❌ 派活给本次清单**没有**的 agent_name
- ❌ 把括号里的中文显示名拼进 `agent_name`
- ❌ 已经达成共识后仍然继续派活
"""
# (id, name, description, soul, skills) — skills 为 None 表示不写 skills 字段。
_BUILTIN_FUNCTIONAL_AGENTS: tuple[tuple[str, str, str, str, list[str] | None], ...] = (
(INTENT_AGENT_ID, INTENT_AGENT_NAME, INTENT_AGENT_DESCRIPTION, INTENT_AGENT_SOUL, None),
(RECOMMENDER_AGENT_ID, RECOMMENDER_AGENT_NAME, RECOMMENDER_AGENT_DESCRIPTION, RECOMMENDER_AGENT_SOUL, None),
(
COORDINATOR_AGENT_ID,
COORDINATOR_AGENT_NAME,
COORDINATOR_AGENT_DESCRIPTION,
COORDINATOR_AGENT_SOUL,
["agent_orchestration"],
),
)
def _ensure_one(agent_id: str, name: str, description: str, soul: str) -> bool:
def _ensure_one(
agent_id: str,
name: str,
description: str,
soul: str,
skills: list[str] | None = None,
) -> bool:
"""Create the on-disk files for one functional agent if missing.
Returns True if files were just created, False if the directory already
existed (and we didn't touch it — respect any local edits).
``skills`` 写入 config.yaml 顶层 ``skills`` 字段,与 routers/agents.py 的
``_write_agent_config`` 输出形状对齐;coordinator 需要 ``agent_orchestration``
才能派活,intent / recommender 不需要任何 skill(传 None 即可)。
"""
agent_dir = get_paths().agent_dir(agent_id)
if agent_dir.exists():
@@ -411,8 +584,13 @@ def _ensure_one(agent_id: str, name: str, description: str, soul: str) -> bool:
# newline="\n" 强制 LF;否则 Windows 下 open() 把 \n 翻成 \r\n,跨 OS 部署时与
# Linux 部署机字节级不一致(功能上 YAML/Markdown 都兼容,但 git diff / 哈希对不齐)。
config_payload: dict[str, object] = {"id": agent_id, "name": name, "description": description}
if skills:
# 与 routers/agents.py::_write_agent_config 行为一致:None 不写、空列表也不写,
# 仅当确有 skill 名时才追加顶层 skills 字段。
config_payload["skills"] = list(skills)
config_yaml = yaml.dump(
{"id": agent_id, "name": name, "description": description},
config_payload,
default_flow_style=False,
allow_unicode=True,
)
@@ -435,9 +613,9 @@ def ensure_roundtable_functional_agents() -> list[str]:
actionable error if the agent really cannot be resolved.
"""
created: list[str] = []
for agent_id, name, description, soul in _BUILTIN_FUNCTIONAL_AGENTS:
for agent_id, name, description, soul, skills in _BUILTIN_FUNCTIONAL_AGENTS:
try:
if _ensure_one(agent_id, name, description, soul):
if _ensure_one(agent_id, name, description, soul, skills):
created.append(agent_id)
except Exception:
logger.exception("Failed to seed roundtable functional agent '%s'", agent_id)
offline-backend-20260512/backend/app/gateway/routers/agents.py
+400 -22
View File
@@ -2,23 +2,31 @@
from __future__ import annotations
import asyncio
import logging
import shutil
import time
from datetime import datetime
from typing import Any
from uuid import uuid4
import yaml
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_agent_store, get_tag_store
from app.gateway.auth.password import verify_password_async
from app.gateway.deps import get_agent_store, get_config, get_tag_store
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.agents_api_config import get_agents_api_config
from deerflow.config.agents_config import list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
from deerflow.config.app_config import AppConfig
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.agents import AgentStore
from deerflow.persistence.tags import TagStore
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.skills.cascade import sync_agent_skills_to_available
from deerflow.skills.storage import get_or_new_skill_storage
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["agents"])
@@ -32,6 +40,7 @@ class AgentResponse(BaseModel):
description: str = Field(default="", description="Agent description")
user_id: str | None = Field(default=None, description="Owner user id. Null means built-in.")
published: bool = Field(default=False, description="Whether a user-created agent is public")
is_private: bool = Field(default=False, description="Whether the agent lives in the password-gated private square instead of the public one")
builtin: bool = Field(default=False, description="True when user_id is null")
created_at: datetime | str | None = None
updated_at: datetime | str | None = None
@@ -50,6 +59,11 @@ class AgentResponse(BaseModel):
description="Admin-curated homepage rank. Null = not featured. Lower sorts first.",
)
is_favorite: bool = Field(
default=False,
description="Whether the requesting user has favorited (收藏) this agent.",
)
class AgentsListResponse(BaseModel):
"""Response model for listing all visible agents."""
@@ -68,6 +82,7 @@ class AgentCreateRequest(BaseModel):
description: str = Field(default="", description="Agent description")
soul: str = Field(default="", description="SOUL.md content - agent personality and behavioral guardrails")
published: bool = Field(default=False, description="Whether the new agent should be public")
is_private: bool = Field(default=False, description="When True, the new agent goes into the password-gated private square instead of the public one")
skills: list[str] | None = Field(default=None, description="Skill whitelist persisted to the agent's config.yaml")
model: str | None = Field(default=None, description="Optional model override persisted to config.yaml")
tool_groups: list[str] | None = Field(default=None, description="Optional tool group whitelist persisted to config.yaml")
@@ -80,6 +95,7 @@ class AgentUpdateRequest(BaseModel):
description: str | None = Field(default=None, description="Updated description")
soul: str | None = Field(default=None, description="Updated SOUL.md content")
published: bool | None = Field(default=None, description="Updated publication state")
is_private: bool | None = Field(default=None, description="Move the agent into / out of the private square (true / false)")
skills: list[str] | None = Field(default=None, description="Updated skill whitelist (config.yaml)")
model: str | None = Field(default=None, description="Updated model override (config.yaml)")
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
@@ -192,6 +208,11 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
This preserves old deployments where agents already existed under
``.deer-flow/agents/{name}`` before the agents table was introduced.
Heavy: scans the agents directory + parses each ``config.yaml`` + issues a
SELECT-then-INSERT/UPDATE per agent. Call once at startup via the lifespan,
and rely on :func:`_maybe_sync_legacy_agents` (TTL-gated) inside request
handlers as a safety net for environments that bypassed the lifespan hook.
"""
try:
for agent_cfg in list_custom_agents():
@@ -210,6 +231,44 @@ async def _sync_legacy_agents(store: AgentStore) -> None:
logger.exception("Failed to sync legacy folder-backed agents")
# Throttle the per-request legacy-sync fallback. The real sync runs once at
# startup (lifespan); request handlers used to call ``_sync_legacy_agents`` on
# every hit, which scanned the filesystem + did N DB upserts per agent each
# time. After the first successful sync we skip the work for ``_LEGACY_SYNC_TTL_SECONDS``
# — long enough to be effectively "startup-only" in steady state, short enough
# that a freshly dropped agent directory gets picked up on the next bucket.
_LEGACY_SYNC_TTL_SECONDS = 3600.0
_legacy_sync_state: dict[str, Any] = {"last_at": 0.0, "lock": None}
async def _maybe_sync_legacy_agents(store: AgentStore) -> None:
"""TTL-gated wrapper around :func:`_sync_legacy_agents`.
Becomes a no-op in steady state — the heavy lifting happens in the lifespan
hook. Inside the TTL window we don't even take the lock; outside it we
serialize callers so a thundering herd doesn't fan out into N filesystem
scans + N*M DB queries.
"""
now = time.monotonic()
if now - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
return
lock = _legacy_sync_state["lock"]
if lock is None:
lock = asyncio.Lock()
_legacy_sync_state["lock"] = lock
async with lock:
# Re-check under the lock — a sibling coroutine may have just refreshed.
if time.monotonic() - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
return
await _sync_legacy_agents(store)
_legacy_sync_state["last_at"] = time.monotonic()
def _mark_legacy_sync_fresh() -> None:
"""Called by the lifespan hook after running :func:`_sync_legacy_agents`."""
_legacy_sync_state["last_at"] = time.monotonic()
def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
"""Lazily restore an agent's on-disk artifacts from its DB record.
@@ -247,20 +306,36 @@ def _record_to_response(
record: dict[str, Any],
*,
include_soul: bool = False,
include_extras: bool = True,
) -> AgentResponse:
_ensure_agent_dir_from_record(record)
"""Convert a DB row to an API response.
Args:
include_soul: Read SOUL.md from disk and embed in ``soul``. Set False
in list endpoints — the frontend (gallery / recommend / selector)
never reads ``soul``, only the edit dialog does. Skipping this
avoids one synchronous file read per agent.
include_extras: Self-heal a missing agent dir and read deprecated
``model`` / ``tool_groups`` / ``skills`` from ``config.yaml``. Set
False in list endpoints to skip a stat + yaml parse per agent —
those fields are unused by current frontends and the dir restore
can happen lazily when the agent is actually opened.
"""
soul: str | None = None
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None}
if include_extras:
_ensure_agent_dir_from_record(record)
extras = _load_config_extras(record["id"])
if include_soul:
soul = load_agent_soul(record["id"]) or ""
extras = _load_config_extras(record["id"])
return AgentResponse(
id=record["id"],
name=record["name"],
description=record.get("description") or "",
user_id=record.get("user_id"),
published=bool(record.get("published", False)),
is_private=bool(record.get("is_private", False)),
builtin=record.get("user_id") is None,
created_at=record.get("created_at"),
updated_at=record.get("updated_at"),
@@ -272,6 +347,25 @@ def _record_to_response(
)
async def _record_to_response_async(
record: dict[str, Any],
*,
include_soul: bool = False,
include_extras: bool = True,
) -> AgentResponse:
"""Async wrapper: when we actually touch disk, hop to a worker thread so
we don't block the asyncio loop. List endpoints (no soul, no extras) skip
this overhead and stay on the calling task."""
if not include_soul and not include_extras:
return _record_to_response(record, include_soul=False, include_extras=False)
return await asyncio.to_thread(
_record_to_response,
record,
include_soul=include_soul,
include_extras=include_extras,
)
async def _get_owned_or_raise(store: AgentStore, agent_id: str, user_id: str) -> dict[str, Any]:
owned = await store.get_owned(agent_id, user_id)
if owned is not None:
@@ -291,17 +385,21 @@ async def _get_editable_or_raise(
) -> tuple[dict[str, Any], str]:
"""Return (record, edit_mode) where edit_mode is 'owner' or 'admin'.
'admin' is granted only when the agent is built-in (user_id IS NULL) and the
caller is a system admin. Custom agents owned by other users still 403.
Admins may edit any agent — built-in (``user_id IS NULL``) or owned by
another user — so the admin console can manage the full agent catalog.
Non-admin callers are still restricted to agents they own.
"""
owned = await store.get_owned(agent_id, user_id)
if owned is not None:
return owned, "owner"
if is_admin:
record = await store.get_any(agent_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
return record, "admin"
visible = await store.get_visible(agent_id, user_id)
if visible is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
if is_admin and visible.get("user_id") is None:
return visible, "admin"
raise HTTPException(status_code=403, detail="You can only modify agents that you created")
@@ -315,20 +413,47 @@ async def list_agents(
request: Request,
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on agent name"),
tag_ids: list[str] | None = Query(default=None, description="Keep agents carrying any of these tags (OR)"),
admin: bool = Query(
default=False,
description=(
"Admin-only override. When ``true`` and the caller is a system admin, "
"every agent is returned (built-in + every user's agents + private-square "
"entries) regardless of the usual ownership/visibility filter."
),
),
) -> AgentsListResponse:
_require_agents_api_enabled()
try:
store = get_agent_store(request)
tag_store: TagStore = get_tag_store(request)
await _sync_legacy_agents(store)
await _maybe_sync_legacy_agents(store)
user_id = _current_user_id(request)
agents = await store.list_visible(user_id)
responses = [_record_to_response(a, include_soul=True) for a in agents]
if admin:
if not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can list all agents")
agents = await store.list_all()
else:
agents = await store.list_visible(user_id)
# Load lightweight extras (skills / model / tool_groups from config.yaml)
# so the gallery cards can render skill badges. SOUL.md is still skipped
# (only the edit dialog reads it). The per-agent config.yaml reads run in
# worker threads concurrently, so the event loop stays responsive even
# with many agents.
responses = list(
await asyncio.gather(
*(
_record_to_response_async(a, include_soul=False, include_extras=True)
for a in agents
)
)
)
assignments = await tag_store.list_assignments("agent", [r.id for r in responses])
favorite_ids = await store.list_favorite_ids(user_id)
for resp in responses:
resp.tags = assignments.get(resp.id, [])
resp.is_favorite = resp.id in favorite_ids
if search:
needle = search.strip().lower()
responses = [r for r in responses if needle in r.name.lower()]
@@ -368,11 +493,18 @@ async def get_agent_selector(request: Request) -> AgentSelectorResponse:
_require_agents_api_enabled()
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
await _maybe_sync_legacy_agents(store)
user_id = _current_user_id(request)
records = await store.list_featured(user_id)
return AgentSelectorResponse(agents=[_record_to_response(r) for r in records])
# Selector dropdown only renders name/description/featured_order — same
# rationale as list_agents: skip soul + extras + self-heal.
return AgentSelectorResponse(
agents=[
_record_to_response(r, include_soul=False, include_extras=False)
for r in records
]
)
except HTTPException:
raise
except Exception as e:
@@ -392,12 +524,18 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
await _maybe_sync_legacy_agents(store)
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
return _record_to_response(record, include_soul=True)
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
# worker thread so the asyncio loop stays responsive under concurrent
# detail fetches.
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
favorite_ids = await store.list_favorite_ids(user_id)
response.is_favorite = agent_id in favorite_ids
return response
except HTTPException:
raise
except Exception as e:
@@ -446,10 +584,11 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
"description": request.description or "",
"user_id": _current_user_id(http_request),
"published": request.published,
"is_private": request.is_private,
}
)
logger.info("Created agent '%s' (%s) at %s", name, agent_id, agent_dir)
return _record_to_response(record, include_soul=True)
return await _record_to_response_async(record, include_soul=True, include_extras=True)
except HTTPException:
raise
@@ -486,6 +625,15 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
update_data["description"] = request.description or ""
if "published" in fields_set:
update_data["published"] = bool(request.published)
if "is_private" in fields_set:
update_data["is_private"] = bool(request.is_private)
# Mutual exclusion: an agent moved into the private square cannot
# also be publicly published. Force ``published=false`` whenever the
# same request sets ``is_private=true``, even if the client forgot
# to clear it. (When the request only sets one of the two, the
# other side is left untouched.)
if update_data.get("is_private") is True:
update_data["published"] = False
next_name = update_data.get("name", current["name"])
next_description = update_data.get("description", current.get("description") or "")
@@ -511,14 +659,17 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
if not update_data:
record = current
elif edit_mode == "admin":
record = await store.update_builtin(agent_id, update_data)
# Admins may edit built-in (``user_id IS NULL``) and any user-owned
# agent — ``update_any`` skips the ownership check that ``update``
# and ``update_builtin`` enforce.
record = await store.update_any(agent_id, update_data)
else:
record = await store.update(agent_id, user_id, update_data)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
logger.info("Updated agent '%s' (%s)", record["name"], agent_id)
return _record_to_response(record, include_soul=True)
return await _record_to_response_async(record, include_soul=True, include_extras=True)
except HTTPException:
raise
@@ -598,7 +749,7 @@ async def delete_agent(agent_id: str, request: Request) -> None:
)
if edit_mode == "admin":
deleted = await store.delete_builtin(agent_id)
deleted = await store.delete_any(agent_id)
else:
deleted = await store.delete(agent_id, user_id)
if not deleted:
@@ -611,6 +762,10 @@ async def delete_agent(agent_id: str, request: Request) -> None:
await get_tag_store(request).unassign_all("agent", agent_id)
except Exception: # noqa: BLE001 — tag cleanup must not block deletion
pass
try:
await store.delete_favorites_for_agent(agent_id)
except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
pass
logger.info("Deleted agent '%s' from %s", agent_id, agent_dir)
except HTTPException:
raise
@@ -619,6 +774,53 @@ async def delete_agent(agent_id: str, request: Request) -> None:
raise HTTPException(status_code=500, detail=f"Failed to delete agent: {str(e)}") from e
@router.post(
"/agents/{agent_id}/favorite",
status_code=204,
summary="Favorite Agent",
description="Bookmark a visible agent into the current user's 我的 tab. Idempotent.",
)
async def favorite_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
await store.add_favorite(user_id, agent_id)
logger.info("User '%s' favorited agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to favorite agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to favorite agent: {str(e)}") from e
@router.delete(
"/agents/{agent_id}/favorite",
status_code=204,
summary="Unfavorite Agent",
description="Remove an agent from the current user's favorites. No-op when not favorited.",
)
async def unfavorite_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
await store.remove_favorite(user_id, agent_id)
logger.info("User '%s' unfavorited agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
@router.put(
"/agents/{agent_id}/featured",
response_model=AgentResponse,
@@ -642,7 +844,7 @@ async def set_agent_featured(
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
await _maybe_sync_legacy_agents(store)
record = await store.get_any(agent_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
@@ -654,7 +856,7 @@ async def set_agent_featured(
if updated is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
logger.info("Set agent '%s' featured_order=%s by admin", agent_id, body.order)
return _record_to_response(updated, include_soul=True)
return await _record_to_response_async(updated, include_soul=True, include_extras=True)
except HTTPException:
raise
except Exception as e:
@@ -662,6 +864,107 @@ async def set_agent_featured(
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
class PrivateSquareListRequest(BaseModel):
"""Request body for listing all private-square agents."""
password: str = Field(..., description="Plain-text private-square password. Verified against the admin-configured hash before the listing is returned.")
class AgentPrivateUpdateRequest(BaseModel):
"""Admin-only request body for moving an agent into / out of the private square."""
is_private: bool = Field(..., description="Set true to move the agent into the private square, false to move it back into the public square.")
@router.post(
"/agents/private-square/list",
response_model=AgentsListResponse,
summary="List Private-Square Agents",
description=(
"Verify the admin-configured private-square password and return every "
"agent flagged ``is_private=true``. Returns 404 when no password is set, "
"403 when the supplied password does not match, and the full agent list "
"otherwise — regardless of ownership, since the password is the gate."
),
)
async def list_private_square(body: PrivateSquareListRequest, request: Request) -> AgentsListResponse:
_require_agents_api_enabled()
settings = load_system_settings().private_square
if not settings.password_hash:
raise HTTPException(status_code=404, detail="Private square password is not configured")
if not body.password or not await verify_password_async(body.password, settings.password_hash):
raise HTTPException(status_code=403, detail="Incorrect private square password")
try:
store = get_agent_store(request)
tag_store: TagStore = get_tag_store(request)
await _sync_legacy_agents(store)
records = await store.list_private()
responses = [_record_to_response(r, include_soul=True) for r in records]
assignments = await tag_store.list_assignments("agent", [r.id for r in responses])
for resp in responses:
resp.tags = assignments.get(resp.id, [])
return AgentsListResponse(agents=responses)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to list private agents: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to list private agents: {str(e)}") from e
@router.put(
"/agents/{agent_id}/private",
response_model=AgentResponse,
summary="Move Agent Between Squares (admin)",
description=(
"Admin-only. Move any agent (built-in or user-owned) into or out of the "
"password-gated private square. Owners can already toggle the flag on "
"their own agents via the regular ``PUT /agents/{id}`` endpoint; this "
"endpoint exists so admins can re-categorize other users' agents too."
),
)
async def set_agent_private(
agent_id: str,
body: AgentPrivateUpdateRequest,
request: Request,
) -> AgentResponse:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
if not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can re-categorize other users' agents")
try:
store = get_agent_store(request)
await _sync_legacy_agents(store)
record = await store.get_any(agent_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
update: dict[str, Any] = {"is_private": bool(body.is_private)}
# Mutual exclusion — see ``update_agent`` above. Moving an agent into
# the private square also drops the public ``published`` flag so we
# never end up with an agent that's "in both squares".
if update["is_private"]:
update["published"] = False
if record.get("user_id") is None:
updated = await store.update_builtin(agent_id, update)
else:
updated = await store.update(agent_id, str(record["user_id"]), update)
if updated is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
logger.info("Admin set agent '%s' is_private=%s", agent_id, body.is_private)
return _record_to_response(updated, include_soul=True)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to set is_private for agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to update agent: {str(e)}") from e
class AgentFeaturedReorderRequest(BaseModel):
"""Request body for admin to bulk-reorder the homepage featured list."""
@@ -705,3 +1008,78 @@ async def reorder_featured_agents(
except Exception as e:
logger.error("Failed to reorder featured agents: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to reorder featured agents: {str(e)}") from e
class AgentSkillRemovalEntry(BaseModel):
agent_id: str = Field(..., description="Agent storage id whose config.yaml was rewritten")
agent_name: str | None = Field(default=None, description="Display name from the DB, if any")
user_id: str | None = Field(default=None, description="Owner user id (null = built-in)")
removed_skills: list[str] = Field(..., description="Skill names that no longer exist on disk and were stripped")
class AgentSkillsSyncResponse(BaseModel):
checked: int = Field(..., description="Number of agent config.yaml files scanned")
modified: int = Field(..., description="Number of agents whose skills list was rewritten")
removals: list[AgentSkillRemovalEntry] = Field(default_factory=list, description="Per-agent removal details")
@router.post(
"/agents/admin/sync-skills",
response_model=AgentSkillsSyncResponse,
summary="Sync Agent Skills (admin)",
description=(
"Admin-only. Scan every agent's ``config.yaml`` and strip skill names that "
"no longer exist on disk. Repairs historical drift where a skill was "
"removed without the delete/unpublish cascade firing (e.g. deleted from "
"the filesystem manually, or removed in an older version of the code)."
),
)
async def admin_sync_agent_skills(
request: Request,
config: AppConfig = Depends(get_config),
) -> AgentSkillsSyncResponse:
_require_agents_api_enabled()
if not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can run agent-skills sync")
try:
storage = get_or_new_skill_storage(app_config=config)
all_skills = storage.load_skills(enabled_only=False)
available = {skill.name for skill in all_skills}
result = await asyncio.to_thread(sync_agent_skills_to_available, available)
# Enrich removals with display name + owner from the DB so the admin
# UI can show "我的智能体 X 移除了技能 Y" instead of just the id.
store = get_agent_store(request)
enriched: list[AgentSkillRemovalEntry] = []
for entry in result.get("removals", []):
agent_id = entry["agent_id"]
record = await store.get_any(agent_id)
enriched.append(
AgentSkillRemovalEntry(
agent_id=agent_id,
agent_name=(record.get("name") if record else None),
user_id=(record.get("user_id") if record else None),
removed_skills=entry["removed_skills"],
)
)
if result.get("modified"):
await refresh_skills_system_prompt_cache_async()
logger.info(
"Admin agent-skills sync: checked=%d modified=%d",
result["checked"],
result["modified"],
)
return AgentSkillsSyncResponse(
checked=result["checked"],
modified=result["modified"],
removals=enriched,
)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to sync agent skills: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to sync agent skills: {str(e)}") from e
offline-backend-20260512/backend/app/gateway/routers/intent.py
+263 -24
View File
@@ -65,6 +65,7 @@ from fastapi.responses import StreamingResponse
# identical in connection handling, payload shape, and error mapping.
from app.gateway.routers._roundtable_seed import ensure_roundtable_functional_agents
from app.gateway.routers.multi_agent import (
_ROUNDTABLE_THREAD_METADATA,
_auth_headers,
_create_thread,
_diagnose_exception,
@@ -93,6 +94,173 @@ _INTENT_READY_PATTERN = re.compile(
)
def _normalize_question(text: str) -> str:
"""归一化澄清问题文本,用于"和过往是否重复"的判断。
LLM 复述时常见模式:
- 完全一样的字符串
- 仅前后空白、标点、emoji 不同(如多/少一个问号)
- 仅大小写差异(中文场景几乎不见,但兼容英文/拼音)
所以归一化时统一:小写 + 去除非字母/数字/中文字符 + 折叠空白。
Levenshtein/语义相似度等模糊匹配先不做,等观察到误判再迭代。
"""
if not isinstance(text, str):
return ""
lowered = text.strip().lower()
# 只保留中英文字符与数字,去掉 ?。!?,。"":/、() 等标点 + emoji + 空白。
return re.sub(r"[^0-9a-z一-鿿]+", "", lowered)
def _normalize_options(raw: object) -> tuple[str, ...]:
"""把 options 拍平成可比较的归一化元组,用于"选项是否和上一轮一致"的判断。"""
if raw is None:
return ()
if isinstance(raw, str):
# 偶有模型把 list 序列化成 JSON 字符串发回来。
try:
raw = json.loads(raw)
except (json.JSONDecodeError, TypeError):
return (_normalize_question(raw),)
if not isinstance(raw, list):
return (_normalize_question(str(raw)),)
out: list[str] = []
for item in raw:
if isinstance(item, dict):
label = item.get("label") or item.get("title") or item.get("name") or item.get("value") or item.get("id") or ""
out.append(_normalize_question(str(label)))
else:
out.append(_normalize_question(str(item)))
# 顺序无关:LLM 偶尔会调换选项顺序,但语义上还是同一组。
return tuple(sorted(o for o in out if o))
def _count_past_clarifications(messages: list[dict[str, Any]]) -> int:
"""统计当前 thread 内已经发出过的 ``ask_clarification`` 次数。
用来给 ``stream_intent`` 做"3 轮硬上限"兜底:
- 0-1 次:正常流程,模型自行决定继续问还是收敛
- 2 次(即将进入第 3 轮):prompt 追加系统提示,要求本轮必须 INTENT_READY
- ≥3 次:工具白名单移除 ``ask_clarification``,模型物理上无法再问
只数"已经成型"的 tool_call,不区分用户是否回答过——LangGraph 把 ask
工具的调用永久记录在 AIMessage.tool_calls 里,数它即可。
"""
if not isinstance(messages, list):
return 0
count = 0
for msg in messages:
if not isinstance(msg, dict):
continue
if (msg.get("type") or "").lower() not in ("ai", "aimessage", "aimessagechunk"):
continue
for tc in msg.get("tool_calls") or []:
if (tc.get("name") or "").lower() == "ask_clarification":
count += 1
return count
async def _fetch_thread_messages(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
thread_id: str,
) -> list[dict[str, Any]]:
"""读取当前 thread checkpoint 的 messages 列表(best-effort)。
用 ASGITransport loopback 调 ``GET /api/threads/{tid}/state``,失败时返回
空列表——计数兜底是个"刹车",失败时退化成原有的纯 SOUL 自律行为,不阻断主流程。
"""
try:
resp = await client.get(f"{base}/api/threads/{thread_id}/state", headers=headers)
except Exception as exc:
logger.warning("intent: get thread state failed (%s) — falling back to soul-only flow", exc)
return []
if resp.status_code != 200:
logger.warning(
"intent: get thread state %s returned %s — falling back to soul-only flow",
thread_id,
resp.status_code,
)
return []
try:
payload = resp.json()
except Exception:
return []
values = payload.get("values") if isinstance(payload, dict) else None
if not isinstance(values, dict):
return []
messages = values.get("messages")
return messages if isinstance(messages, list) else []
# 硬上限:超过这个轮数,无论模型怎么想,都强制移除 ask_clarification 工具。
# 与 SOUL.md §0 「硬规则 2」对齐——"绝不存在第 4 轮"。
_MAX_CLARIFICATION_ROUNDS = 3
def _is_repeated_clarification(
messages: list[dict[str, Any]],
*,
new_question: str,
new_options: object,
new_tool_call_id: str | None = None,
) -> bool:
"""检测本轮的 ask_clarification 是否在过往轮次问过。
判定为重复必须同时满足:
1. 历史里至少有一次 ``ask_clarification`` 的问题文本 / 选项 与本次归一化相同;
2. 用户在那次之后**已经发过消息**(即"已回答过")。
没有这条限制会把"模型在同一 run 内多次输出同一 tool_call_chunks"
误判成重复,导致首次澄清都吞掉。
3. 不是同一个 tool_call(LangGraph 偶有把同一 call 拆成多块返回的情况)。
传回 True → 上层应把 status 从 ``clarification`` 降级为 ``asking``,
避免前端弹同一张卡片(配合 SOUL §2.5 prompt 双层防御)。
"""
norm_new_q = _normalize_question(new_question)
norm_new_opts = _normalize_options(new_options)
if not norm_new_q and not norm_new_opts:
return False
seen_match_index: int | None = None
for idx, msg in enumerate(messages):
if not isinstance(msg, dict):
continue
if (msg.get("type") or "").lower() not in ("ai", "aimessage", "aimessagechunk"):
continue
for tc in msg.get("tool_calls") or []:
if (tc.get("name") or "").lower() != "ask_clarification":
continue
if new_tool_call_id and tc.get("id") == new_tool_call_id:
# 同一个 tool_call 的不同 chunk,不算重复。
continue
args = tc.get("args") or {}
past_q = _normalize_question(str(args.get("question") or ""))
past_opts = _normalize_options(args.get("options"))
# 问题文本和选项任一明确重叠就视为重复:
# - question 相同(options 可能空)
# - options 相同且非空(question 可能被改写)
q_match = bool(past_q) and past_q == norm_new_q
opt_match = bool(past_opts) and past_opts == norm_new_opts
if q_match or opt_match:
seen_match_index = idx
break
if seen_match_index is not None:
break
if seen_match_index is None:
return False
# 校验用户在该次澄清之后是否已经发过 human 消息(= 回答过)。
for msg in messages[seen_match_index + 1:]:
if not isinstance(msg, dict):
continue
msg_type = (msg.get("type") or "").lower()
if msg_type in ("human", "humanmessage", "user"):
return True
return False
@router.post("/init")
async def init_intent(request: Request, payload: dict = Body(...)) -> dict[str, Any]:
"""创建任务理解会话用的空 thread。
@@ -119,7 +287,7 @@ async def init_intent(request: Request, payload: dict = Body(...)) -> dict[str,
# 看不到逐 token 增量。
try:
async with _make_loopback_client(request) as client:
thread_id = await _create_thread(client, base, headers)
thread_id = await _create_thread(client, base, headers, metadata=_ROUNDTABLE_THREAD_METADATA)
except HTTPException:
# 上游已用结构化方式失败,原样传给前端不需要二次包装。
raise
@@ -176,33 +344,70 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
# 因此既无须 trust_env=False 防代理，也不存在 loopback 40s 超时这种坑。
async with _make_loopback_client(request) as client:
try:
# —— 3 轮硬上限兜底 ——
# 在发起 run 之前先读一次 thread state,数当前 ask_clarification 已发出次数:
# past=0/1 → 正常,不动 message,不动工具白名单
# past=2(本轮即将变成第 3 轮)→ 给消息加前缀,强提示"必须 INTENT_READY"
# past≥3 → 直接把 ask_clarification 也加进 excluded_tools,工具物理消失
# 这一步是个"刹车":SOUL 让模型自律,这里让系统兜底。
past_clarifications = _count_past_clarifications(
await _fetch_thread_messages(client, base, headers, thread_id)
)
# 1. ask_clarification 的工具白名单——超过硬上限就移除。
base_excluded_tools = [
"web_search",
"present_files",
"view_image",
"agent_orchestration",
"bash",
"ls",
"read_file",
"write_file",
"str_replace",
"memory",
"hindsight_recall",
"hindsight_reflect",
"hindsight_retain",
"task",
"write_todos",
]
if past_clarifications >= _MAX_CLARIFICATION_ROUNDS:
base_excluded_tools.append("ask_clarification")
logger.info(
"intent stream: past_clarifications=%d >= %d, force-disabling "
"ask_clarification to break the loop",
past_clarifications,
_MAX_CLARIFICATION_ROUNDS,
)
# 2. message 前缀——给模型一个明确的"本轮必须收敛"信号。
effective_message = message
if past_clarifications >= _MAX_CLARIFICATION_ROUNDS:
effective_message = (
f"[系统强制] 已澄清 {past_clarifications} 轮,触发硬上限。"
f"本轮请直接输出 [INTENT_READY],缺的字段用 assumptions 兜底,"
f"绝对不要再调任何工具,也不要再询问任何问题。\n\n"
f"用户最新输入:{message}"
)
elif past_clarifications >= _MAX_CLARIFICATION_ROUNDS - 1:
effective_message = (
f"[系统提示] 已澄清 {past_clarifications} 轮,这是最后一轮。"
f"本轮请直接输出 [INTENT_READY],未明确的字段用 assumptions 兜底。\n\n"
f"用户最新输入:{message}"
)
body = _run_payload(
agent_name=INTENT_AGENT_ID,
model_name=model_name,
thread_id=thread_id,
new_message=message,
# Intent agent only ever needs ask_clarification. Block
new_message=effective_message,
# Intent agent only ever needs ask_clarification (and even that
# is yanked once past_clarifications hits the cap above). Block
# everything else so the model can't drift into solutioning
# (e.g. saving a design doc with write_file) — the model
# tends to ignore SOUL prose, hard exclusion is reliable.
# 仅保留 ask_clarification；其余工具见模块 docstring「工具白名单」。
excluded_tools=[
"web_search",
"present_files",
"view_image",
"agent_orchestration",
"bash",
"ls",
"read_file",
"write_file",
"str_replace",
"memory",
"hindsight_recall",
"hindsight_reflect",
"hindsight_retain",
"task",
"write_todos",
],
# (e.g. saving a design doc with write_file) — SOUL prose is
# not reliable on its own, hard exclusion is.
excluded_tools=base_excluded_tools,
skill_stop_names=[],
)
@@ -235,7 +440,13 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
except json.JSONDecodeError as exc:
logger.warning("intent summary JSON parse failed: %s", exc)
summary = None
ready_text = msg_text
# SOUL §0 硬规则 3 要求 ```json``` 反引号闭合后不得再输出任何字符,
# 但模型仍偶尔会在后面接 "那么我帮你列一下大纲……" 之类的尾巴。
# 终态帧只回到 marker 块结尾 + 一个换行,把废话从 content 里裁掉——
# 草稿持久化 / 跨设备重放 / 前端 splitIntentReadyBlock 的 after
# 段都因此干净。流式期间用户仍会逐 token 看到那段废话,前端
# Step1Panel 已经把 `parts.after` 隐藏掉,体验闭环。
ready_text = msg_text[: m.end()].rstrip() + "\n"
break
if summary and ready_text:
final_frame = {
@@ -264,6 +475,34 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
# 比 tool_call 旁的短 preamble 更适合作为气泡正文。
follow_up = _find_trailing_ai_text(messages, exclude_text=content)
display_content = follow_up or content or question
# 兜底:模型偶尔会复述上一轮的同一澄清问题(SOUL §2.5 prompt
# 已经强约束,但小模型 / 长上下文仍可能漏跟随)。检测到完全
# 重复时,把 status 从 clarification 降级为 asking,前端只渲染
# 文本气泡不弹卡片,避免用户看到"同样选项又出现一次"。
if _is_repeated_clarification(
messages,
new_question=question,
new_options=cargs.get("options"),
new_tool_call_id=clarification_call.get("id"),
):
logger.warning(
"intent stream: dropping duplicate ask_clarification "
"(question=%r) — already asked & answered earlier in thread",
question[:80],
)
fallback_msg = (
display_content
or "(系统识别到智能体重复了上一问题,已自动跳过。请补充其他信息继续推进。)"
)
final_frame: dict[str, Any] = {
"status": "asking",
"content": fallback_msg,
"summary": None,
}
yield f"data: {json.dumps(final_frame, ensure_ascii=False)}\n\n"
return
final_frame: dict[str, Any] = {
"status": "clarification",
"content": display_content,
offline-backend-20260512/backend/app/gateway/routers/mcp.py
+7 -4
View File
@@ -182,10 +182,13 @@ async def update_mcp_configuration(request: McpConfigUpdateRequest, http_request
current_config = get_extensions_config()
# Convert request to dict format for JSON serialization
config_data = {
"mcpServers": {name: server.model_dump() for name, server in request.mcp_servers.items()},
"skills": {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()},
}
config_data = {
"mcpServers": {name: server.model_dump(by_alias=True) for name, server in request.mcp_servers.items()},
"skills": {
name: skill.model_dump(by_alias=True, exclude_none=True)
for name, skill in current_config.skills.items()
},
}
# Write the configuration to file
with open(config_path, "w", encoding="utf-8") as f:
offline-backend-20260512/backend/app/gateway/routers/multi_agent.py
+302 -121
View File
@@ -53,18 +53,32 @@ import os
import re
import time
import traceback
import uuid
from typing import Any, AsyncIterator
import httpx
from app.gateway.routers._roundtable_seed import (
COORDINATOR_AGENT_ID,
ensure_roundtable_functional_agents,
)
from deerflow.config import get_app_config
from deerflow.tools.builtins.clarification_utils import resolve_allow_multiple
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import now_iso
from fastapi import APIRouter, Body, HTTPException, Request
from fastapi.responses import StreamingResponse
from langgraph.checkpoint.base import empty_checkpoint
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/multi-agent", tags=["multi-agent"])
# 圆桌（intent / recommend / 席位 / 协调）创建的 thread 都是功能型内部线程,
# 不是用户的普通对话。打上这个 metadata,前端「最近的对话」列表(recent-chat-list.tsx)
# 据 `metadata.thread_type` 过滤掉,与 scheduler / bootstrap 系统会话同一套机制。
_ROUNDTABLE_THREAD_METADATA: dict[str, Any] = {"thread_type": "roundtable", "system": True}
# ---------------------------------------------------------------------------
# 共享:异常诊断器(intent/recommend/multi-agent 共用)
# ---------------------------------------------------------------------------
@@ -285,9 +299,22 @@ def _auth_headers(request: Request) -> dict[str, str]:
return {"Authorization": token, "Content-Type": "application/json"}
async def _create_thread(client: httpx.AsyncClient, base: str, headers: dict[str, str]) -> str:
"""``POST /api/threads`` → 新 thread_id（intent/recommend/init 席位均用）。"""
resp = await client.post(f"{base}/api/threads", json={}, headers=headers)
async def _create_thread(
client: httpx.AsyncClient,
base: str,
headers: dict[str, str],
*,
metadata: dict[str, Any] | None = None,
) -> str:
"""``POST /api/threads`` → 新 thread_id（intent/recommend/init 席位均用）。
``metadata`` 写入 thread 元数据,圆桌调用方传 ``_ROUNDTABLE_THREAD_METADATA``
把这些功能型线程标记成可被前端「最近的对话」过滤掉的系统会话。
"""
body: dict[str, Any] = {}
if metadata:
body["metadata"] = metadata
resp = await client.post(f"{base}/api/threads", json=body, headers=headers)
resp.raise_for_status()
return resp.json()["thread_id"]
@@ -323,6 +350,110 @@ async def _create_agent(
return False, resp.text
# ---------------------------------------------------------------------------
# 同进程直调 helpers(仅供 init_multi_agent 用,不要扩散到其它地方)
#
# init 阶段串了 N+1 个 thread create + N 个 GET /api/agents + 1 个 POST /api/agents,
# 用 HTTP loopback 串走每次都得 TCP + 鉴权中间件 + Pydantic 反序列化一遍。每个
# 调用 10-30ms,加起来对前端就是「点进 Step 2 后席位灯一颗一颗亮」的卡顿来源。
#
# 这里把这三个调用改成直接吃 app.state 里的 store/checkpointer —— 跑在同一个事
# 件循环里,省掉 TCP + 序列化 + 中间件链。鉴权 / 校验 / 文件副作用按需在直调
# helper 里显式做掉,不要"绕过"任何一项。
#
# 设计边界:这里只覆盖 init 阶段。SSE 长连接(/runs/stream / state)留在 HTTP
# loopback,因为它们本就是别的进程模型(LangGraph 子树),抽出来风险大收益小。
# ---------------------------------------------------------------------------
def _resolve_user_id(request: Request) -> str:
"""与 routers/agents._current_user_id 完全等价,直调路径里用。"""
user = getattr(request.state, "user", None)
if user is not None:
return str(user.id)
return get_effective_user_id()
async def _create_thread_direct(request: Request, *, metadata: dict[str, Any] | None = None) -> str:
"""同进程版的 ``POST /api/threads``。
完全镜像 ``routers/threads.create_thread`` 的写盘动作(thread_meta +
空 checkpoint),但跳过 TCP + Pydantic 反序列化。仅供 init 阶段批量建席位
线程用,失败时抛 HTTPException 由 SSE 帧上报。
``metadata`` 写入 thread 元数据;init 传 ``_ROUNDTABLE_THREAD_METADATA`` 把
席位 / 协调线程标记成系统会话,从前端「最近的对话」里隐藏。
"""
from app.gateway.deps import get_checkpointer, get_thread_store
checkpointer = get_checkpointer(request)
thread_store = get_thread_store(request)
thread_id = str(uuid.uuid4())
now = now_iso()
try:
await thread_store.create(thread_id, assistant_id=None, metadata=metadata or {})
except Exception:
logger.exception("[init-direct] thread_meta create failed for %s", thread_id)
raise HTTPException(status_code=500, detail="Failed to create thread")
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
ckpt_metadata = {
"step": -1,
"source": "input",
"writes": None,
"parents": {},
"created_at": now,
}
try:
await checkpointer.aput(config, empty_checkpoint(), ckpt_metadata, {})
except Exception:
logger.exception("[init-direct] checkpoint create failed for %s", thread_id)
raise HTTPException(status_code=500, detail="Failed to create thread")
return thread_id
async def _get_agents_batch_direct(
request: Request,
agent_ids: list[str],
) -> dict[str, dict[str, Any]]:
"""同进程版的 N 个并行 ``GET /api/agents/{id}`` 合并为 1 次 DB 查询。
返回 ``{agent_id: {id, name, description}}``。**严格模式**:任一席位对当前
用户不可见就抛 404 —— 与原 HTTP loopback 的 ``_get_agent`` 行为一致(原路径
单查 404 会 raise,在外层被 except 捕获并 yield error 帧),不要悄悄回退。
"""
from app.gateway.deps import get_agent_store
store = get_agent_store(request)
user_id = _resolve_user_id(request)
records = await store.list_by_ids(agent_ids, user_id)
out: dict[str, dict[str, Any]] = {}
for r in records:
rid = str(r.get("id") or "").lower()
if not rid:
continue
out[rid] = {
"id": rid,
"name": str(r.get("name") or rid),
"description": str(r.get("description") or ""),
}
missing = [sid for sid in agent_ids if sid.lower() not in out]
if missing:
raise HTTPException(
status_code=404,
detail=f"Agent(s) not visible to current user: {', '.join(missing)}",
)
return out
# NOTE: 历史上这里曾有 _create_coordinator_direct(),每次 init 都创建一个带时间戳的
# roundtable-coordinator-<ts36>。2026-05 改造为单例(见 _roundtable_seed.py 与本
# 模块 init_multi_agent 的 docstring),不再需要"每次创建"的直调辅助;改造后由
# ensure_roundtable_functional_agents() 在路由入口落地静态 SOUL,席位列表通过
# _append_thread_message 注入到 coordinator thread 的对话历史中。
async def _append_thread_message(
client: httpx.AsyncClient,
base: str,
@@ -456,6 +587,8 @@ def _run_payload(
new_message: str,
excluded_tools: list[str],
skill_stop_names: list[str],
thinking_enabled: bool = True,
reasoning_effort: str = "medium",
) -> dict[str, Any]:
"""构造 ``POST /api/threads/{thread_id}/runs/stream`` 的请求体。
@@ -463,6 +596,10 @@ def _run_payload(
- ``excluded_tools``：按路由裁剪工具面（intent 仅澄清、special 禁派活等）。
- ``skill_stop_names``：leader 侧配合 SkillStopMiddleware 在 ``agent_orchestration``
调用后截断图，以便网关从 AIMessage.tool_calls 解析派活列表。
- ``thinking_enabled`` / ``reasoning_effort``：默认开 CoT + medium，适合 Step 1 澄清
与子智能体交付这类需要推理的场景。**推荐弹窗与总控派活**是短决策任务，调用方
应传 ``thinking_enabled=False`` + ``reasoning_effort="low"``，省掉模型先吐
reasoning tokens 才开口的时间，显著降低前端"调度中"首字节延迟。
"""
return {
"input": {
@@ -479,11 +616,17 @@ def _run_payload(
"agent_name": agent_name,
"model_name": model_name,
"mode": "pro",
"reasoning_effort": "medium",
"thinking_enabled": True,
"reasoning_effort": reasoning_effort,
"thinking_enabled": thinking_enabled,
"is_plan_mode": False,
"subagent_enabled": False,
"thread_id": thread_id,
# 圆桌四个 router(intent / recommend / leader / special)都是临时任务,
# 用户个人记忆与本次研讨无关。MemoryMiddleware 的 Hindsight recall 是
# 同步阻塞 HTTP 调用(单次 timeout 15s),如果记忆服务慢/不通会让前端
# "调度中"卡很久;这里默认跳过,builtin USER.md/MEMORY.md 仍通过系统
# 提示注入不受影响。
"memory_recall_disabled": True,
},
"stream_mode": ["messages-tuple", "values"],
"stream_subgraphs": True,
@@ -508,137 +651,165 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
{
"agent_names": ["roundtable-intelligence", ...], // 用户选的席位 id
"main_agent_name": "roundtable-coordinator-xxx", // 当次总控 id，前端生成
"model": "<可选>" // 可选,写入 coordinator agent;不传由 lead_agent 回退到 config.yaml models[0]
"main_agent_name": "<可选,已废弃>", // 历史兼容字段,新代码忽略,统一用 COORDINATOR_AGENT_ID
"model": "<可选>" // 可选;不传由 lead_agent 回退到 config.yaml models[0]
}
SSE 事件（每条 ``data: {...}\\n\\n``）::
seat_ready — 某一席位 thread 已创建（``asyncio.as_completed`` 顺序，先完先发）
coordinator_ready — 总控 agent + thread 已就绪
coordinator_ready — 总控 thread 就绪 + 已注入本次席位列表
init_done — ``thread_ids`` 全量映射，前端存 ``threadIds`` 并开始编排
error — 失败详情（同步校验失败则直接 HTTP 4xx，不进 SSE）
协调智能体 SOUL 在运行时根据各席位 ``GET /api/agents/{id}`` 的 name/description 拼接，
强调 ``agent_name`` 必须用英文 id（``roundtable-intelligence``），禁止带中文括号后缀。
协调智能体改造说明
----------------
协调智能体现在是 **单例**(id 固定为 ``roundtable-coordinator``,SOUL 见
``_roundtable_seed.COORDINATOR_AGENT_SOUL``),由 ``ensure_roundtable_functional_agents()``
在首次调用时自动落地。每次 init **不再创建新的 coordinator agent**(避免
``.deer-flow/agents/`` 累积 ``roundtable-coordinator-<ts36>`` 残留)。
本次圆桌的「可调度席位列表」由 init 在创建好 coordinator thread 之后,以一条
human 消息追加到 thread 历史里,leader run 每轮都能从对话历史里读到。这样
SOUL 长期稳定,席位列表随会话动态注入。前端为兼容旧 client 仍可传
``main_agent_name`` 字段,但后端会忽略并使用 ``COORDINATOR_AGENT_ID``。
"""
# 调用前自检:与 intent/recommend 一样,缺失时就地补建,避免 /run/stream 报 500。
ensure_roundtable_functional_agents()
agent_names_in = payload.get("agent_names") or []
agent_names: list[str] = [str(name).lower() for name in agent_names_in]
main_agent_name: str = str(payload.get("main_agent_name") or "main-agent").lower()
model: str | None = payload.get("model")
# main_agent_name 是历史字段,2026-05 改造后忽略,统一用单例 id。保留接收但不再用前端值,
# 避免下游再次创建带时间戳的临时 coordinator。
main_agent_name = COORDINATOR_AGENT_ID
if not agent_names:
raise HTTPException(status_code=400, detail="agent_names is required")
for name in agent_names:
_validate_agent_id(name, "agent_names entry")
_validate_agent_id(main_agent_name, "main_agent_name")
base = _loopback_base(request)
headers = _auth_headers(request)
def _frame(payload: dict[str, Any]) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
async def stream_generator() -> AsyncIterator[str]:
# TCP loopback 到 127.0.0.1:<gateway port>。client 配置见 _make_loopback_client：
# trust_env=False 防 HTTP_PROXY 拦截,timeout=None 容纳 SSE 长连接。
async with _make_loopback_client(request) as client:
# init 阶段 fast path:全部走同进程直调(_create_thread_direct /
# _get_agents_batch_direct / _create_coordinator_direct),不再走 HTTP
# loopback —— TCP + 鉴权中间件 + Pydantic 反序列化每次 10-30ms,N+1 个席位
# 累计就是"点进 Step 2 后席位一颗颗亮"的卡顿来源。
try:
t_init_start = time.perf_counter()
# 1) 一次 DB 查询拿到所有席位的 name/description,替代之前的 N 个并行
# GET /api/agents/{id}。SOUL 拼接需要描述,这里一次性拿齐。
agent_meta = await _get_agents_batch_direct(request, agent_names)
t_meta_done = time.perf_counter()
# 2) 每个席位 thread 并行创建(直调 thread_store + checkpointer),
# 用 asyncio.as_completed 保持"先完先发 seat_ready"的体验。
async def _prepare_sub_agent(sub_id: str) -> tuple[str, str, str, str]:
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
meta = agent_meta.get(sub_id.lower(), {})
return (
sub_id.lower(),
thread_id,
str(meta.get("name") or sub_id),
str(meta.get("description") or ""),
)
tasks = [asyncio.create_task(_prepare_sub_agent(sid)) for sid in agent_names]
prepared: list[tuple[str, str, str, str]] = []
try:
# Per-seat prepare task — same body as before, but we no longer
# await them with asyncio.gather. Instead, asyncio.as_completed
# surfaces them in REAL completion order so we can emit a
# seat_ready frame the moment each finishes.
async def _prepare_sub_agent(sub_id: str) -> tuple[str, str, str, str]:
info = await _get_agent(client, base, headers, sub_id)
display_name = str(info.get("name") or sub_id)
description = info.get("description", "") or ""
thread_id = await _create_thread(client, base, headers)
return sub_id.lower(), thread_id, display_name, description
for fut in asyncio.as_completed(tasks):
sid, thread_id, display_name, description = await fut
prepared.append((sid, thread_id, display_name, description))
yield _frame({
"event": "seat_ready",
"agent_name": sid,
"thread_id": thread_id,
"display_name": display_name,
"description": description,
})
except Exception:
# Cancel any still-pending sibling so partially-prepared seats
# don't keep running past the stream close.
for t in tasks:
if not t.done():
t.cancel()
raise
t_seats_done = time.perf_counter()
tasks = [asyncio.create_task(_prepare_sub_agent(sid)) for sid in agent_names]
prepared: list[tuple[str, str, str, str]] = []
try:
for fut in asyncio.as_completed(tasks):
sid, thread_id, display_name, description = await fut
prepared.append((sid, thread_id, display_name, description))
yield _frame({
"event": "seat_ready",
"agent_name": sid,
"thread_id": thread_id,
"display_name": display_name,
"description": description,
})
except Exception:
# Cancel any still-pending sibling so we don't leak tasks
# past the AsyncClient close.
for t in tasks:
if not t.done():
t.cancel()
raise
agent_threads: dict[str, str] = {sid: thread_id for sid, thread_id, _, _ in prepared}
seat_lines: list[str] = [
f"- agent_name: {sid}（{display_name}）：{description}"
for sid, _, display_name, description in prepared
]
agent_threads: dict[str, str] = {sid: thread_id for sid, thread_id, _, _ in prepared}
descriptions: list[str] = [
f"- agent_name: {sid}（{display_name}）：{description}"
for sid, _, display_name, description in prepared
]
# 协调智能体改造为**单例**(id = roundtable-coordinator):
# - SOUL 已经由 ensure_roundtable_functional_agents() 在路由入口落地,
# 通用协调规则常驻磁盘,无需每次重写。
# - 本次"可调度席位清单"以 human 消息形式 append 到 coordinator
# thread 历史里,leader run 每轮都能在对话历史最前面看到。
# 这样席位列表随会话动态变化,但磁盘上不再累积 roundtable-coordinator-<ts36>。
#
# 注意:model 字段在 lead_agent run 时通过 context.model_name 注入,
# 不需要写回到 agent config.yaml(否则会污染单例的配置)。
seat_introduction = (
"【圆桌系统初始化】本次研讨可调度的席位清单如下,请严格按 agent_name "
"通过 agent_orchestration 技能派活,不要派给不在清单内的 agent:\n"
+ "\n".join(seat_lines)
)
# 动态 SOUL：把本次 init 的席位列表写进总控 prompt，供 agent_orchestration 派活。
soul = (
"#核心定位：\n"
"用途：根据用户的任务目标,调用其他智能体干活,只负责协调汇总,"
"当你觉得已有信息支持回答用户问题时候,停止调度,并输出最后的结果\n"
"#调度方法：\n"
"通过使用agent_orchestration技能调用其他智能体干活,使用技能时传入agent_name和需要做的活。"
"注意：agent_name 必须严格使用下方列出的英文 id（例如 roundtable-intelligence,"
"或形如 02714cbe6e64406a8a4659f0e3d9c0b8 这样的 UUID）,"
"不要把括号里的中文显示名拼进 agent_name 里。\n"
"#关键约束(必读)：\n"
"1. **不要对同一子智能体重复派同样的活**：每次派活前回顾对话历史,"
"如果某个子智能体已经针对你正要派的任务完成过交付("
"看到形如\"子智能体[名称]完成XX工作,交付内容为:...\"的 human 消息),"
"就直接基于已有交付推进,**不要重复派活**,除非你确实有**具体的、不一样的**补充问题。\n"
"2. **看到交付要会用**：子智能体的交付内容会作为 human 消息追加到你的对话历史里,"
"你能完整看见。当各方信息已足够形成结论时,**直接综合输出最终方案**(此时不再调用任何工具),"
"让讨论收敛。\n"
"3. **避免无意义循环**：如果连续两轮都没有新信息流入,立刻停止派活并基于现有素材给出最终结论。\n"
"#可调度智能体：\n"
+ "\n".join(descriptions)
)
coord_thread = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
agent_threads[main_agent_name] = coord_thread
coord_thread = await _create_thread(client, base, headers)
agent_threads[main_agent_name] = coord_thread
ok, result = await _create_agent(
client,
base,
headers,
name=main_agent_name,
description="你是总控控制协调智能体,负责协调其他智能体干活",
model=model,
skills=["agent_orchestration"],
soul=soul,
agent_id=main_agent_name,
)
if not ok and isinstance(result, str) and "exists" not in result:
yield _frame({"event": "error", "detail": f"create coordinator agent failed: {result}"})
return
# 把席位列表注入 coordinator thread 的对话历史。HTTP loopback 调用
# 走 /api/threads/{tid}/state(读+写),开销 ~10-30ms,在 init 整体
# 几百 ms 的尺度上微不足道,但比起手撕 langgraph checkpoint 内部结构
# 风险低、与 _broadcast / _append_thread_message 一套语义,长期可维护。
try:
base = _loopback_base(request)
headers = _auth_headers(request)
async with _make_loopback_client(request) as seed_client:
await _append_thread_message(
seed_client,
base,
headers,
coord_thread,
seat_introduction,
)
except Exception:
# 注入失败不阻断 init:leader run 仍能跑,只是看不到席位清单,
# 可能派活给错的 id。会被 _strip_display_suffix / _validate_agent_id
# 兜底,前端能看到错误帧;运维通过 backend 日志找到根因再修复。
logger.exception("[init-direct] failed to seed coordinator thread with seat list")
yield _frame({
"event": "coordinator_ready",
"main_agent_name": main_agent_name,
"thread_id": coord_thread,
})
yield _frame({
"event": "init_done",
"main_agent_name": main_agent_name,
"thread_ids": agent_threads,
})
except HTTPException as exc:
logger.warning("multi-agent init HTTPException: %s %s", exc.status_code, exc.detail)
yield _frame({"event": "error", "detail": f"[{exc.status_code}] {exc.detail}"})
except Exception as exc:
logger.exception("multi-agent init unexpected failure")
yield _frame({"event": "error", "detail": f"unexpected: {exc!r}"})
t_coord_done = time.perf_counter()
logger.info(
"[init-direct] meta=%.3fs seats=%.3fs coord=%.3fs total=%.3fs seats_count=%d",
t_meta_done - t_init_start,
t_seats_done - t_meta_done,
t_coord_done - t_seats_done,
t_coord_done - t_init_start,
len(prepared),
)
yield _frame({
"event": "coordinator_ready",
"main_agent_name": main_agent_name,
"thread_id": coord_thread,
})
yield _frame({
"event": "init_done",
"main_agent_name": main_agent_name,
"thread_ids": agent_threads,
})
except HTTPException as exc:
logger.warning("multi-agent init HTTPException: %s %s", exc.status_code, exc.detail)
yield _frame({"event": "error", "detail": f"[{exc.status_code}] {exc.detail}"})
except Exception as exc:
logger.exception("multi-agent init unexpected failure")
yield _frame({"event": "error", "detail": f"unexpected: {exc!r}"})
return StreamingResponse(
stream_generator(),
@@ -764,25 +935,30 @@ def _find_trailing_ai_text(messages: list[dict[str, Any]], *, exclude_text: str
def _find_dispatch_message(messages: list[dict[str, Any]]) -> dict[str, Any]:
"""Locate the latest AIMessage that carries dispatch tool calls.
"""Locate the latest AIMessage from THIS turn.
The very last message can be a skill-stop ToolMessage injected by
``SkillStopMiddleware`` when it intercepts the dispatch, or a clarification
ToolMessage. Either way, the actionable tool_calls live on the AIMessage
*before* it, so we walk backwards until we hit one with ``tool_calls``.
Returns the latest AIMessage when none of them carry tool_calls so the
caller can still extract its text content.
Trailing ToolMessages (skill_stop / clarification ack injected by
``SkillStopMiddleware`` or ``ClarificationMiddleware``) are skipped because
the actionable tool_calls live on the AIMessage that emitted them. We stop
at the **first AIMessage encountered** — its ``tool_calls`` (or lack
thereof) is the authoritative signal for this turn.
历史踩坑:之前实现是"一直往前翻直到撞到带 tool_calls 的 AI",看似优雅,
实际上让 leader 永远收敛不了:本轮 LLM 不论怎么共识,只要历史上派过活,
上一轮的派活 AI 就会被翻出来再用一次,前端会把同样的 task 又发给子智能体,
陷入死循环。现在停在最新一条 AI 上,把判定交给调用方:`tool_calls=[]` →
共识分支,有 tool_calls → 派活分支。
"""
last_ai: dict[str, Any] = {}
for msg in reversed(messages):
if not isinstance(msg, dict):
continue
if (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
if not last_ai:
last_ai = msg
if msg.get("tool_calls"):
return msg
return last_ai
msg_type = (msg.get("type") or "").lower()
if msg_type in ("tool", "toolmessage"):
continue # 跳过末尾 skill_stop / clarification ack
if msg_type in ("ai", "aimessage", "aimessagechunk"):
return msg
break # 撞到 Human/其他类型,本轮没 AI(异常路径,留空让上游报错)
return {}
async def _leader_run(
@@ -820,6 +996,11 @@ async def _leader_run(
new_message=new_message,
excluded_tools=["web_search"],
skill_stop_names=effective_stop_names,
# 总控派活是"下一步派谁、什么任务"的短决策，不需要 CoT。
# 关掉 thinking + low reasoning 让"调度中"几乎瞬间转入"正在派活"，
# 子智能体仍保留 thinking（在 _special_run 里默认 True），保证交付质量。
thinking_enabled=False,
reasoning_effort="low",
)
t_payload_built = time.perf_counter()
logger.info(
offline-backend-20260512/backend/app/gateway/routers/recommend.py
+8 -1
View File
@@ -49,6 +49,7 @@ from fastapi.responses import StreamingResponse
from app.gateway.routers._roundtable_seed import ensure_roundtable_functional_agents
from app.gateway.routers.multi_agent import (
_ROUNDTABLE_THREAD_METADATA,
_auth_headers,
_create_thread,
_flatten_content,
@@ -237,7 +238,7 @@ async def stream_recommend(request: Request, payload: dict = Body(...)) -> Strea
async with _make_loopback_client(request) as client:
try:
# 每次推荐独立 thread；prompt 已包含完整 intent + 候选列表，无需多轮。
thread_id = await _create_thread(client, base, headers)
thread_id = await _create_thread(client, base, headers, metadata=_ROUNDTABLE_THREAD_METADATA)
body = _run_payload(
agent_name=RECOMMENDER_AGENT_ID,
@@ -253,6 +254,12 @@ async def stream_recommend(request: Request, payload: dict = Body(...)) -> Strea
"ask_clarification",
],
skill_stop_names=[],
# 推荐是"从候选池挑 2-6 个"的短分类任务，不需要 CoT。
# 关掉 thinking + 用 low reasoning，避免模型先吐数千 reasoning
# tokens 才开始输出 [RECOMMEND_READY]，前端"等待推荐结果..."
# 从 10-30s 降到 2-5s。
thinking_enabled=False,
reasoning_effort="low",
)
captured: list[str] = []
offline-backend-20260512/backend/app/gateway/routers/scheduled_tasks.py
+6
View File
@@ -37,6 +37,10 @@ class ScheduledTaskCreateRequest(BaseModel):
timezone: str | None = None
execution_agent_name: str | None = None
execution_context: dict[str, Any] | None = None
require_markdown: bool = Field(
default=False,
description="Require the result to be Markdown and exceed the minimum length, else mark the run failed.",
)
class ScheduledTaskUpdateRequest(BaseModel):
@@ -48,6 +52,7 @@ class ScheduledTaskUpdateRequest(BaseModel):
enabled: bool | None = None
execution_agent_name: str | None = None
execution_context: dict[str, Any] | None = None
require_markdown: bool | None = None
class ScheduledTaskDeliveryProfileUpdateRequest(BaseModel):
@@ -555,6 +560,7 @@ async def create_scheduled_task(body: ScheduledTaskCreateRequest, request: Reque
timezone=body.timezone,
execution_agent_name=body.execution_agent_name,
execution_context=body.execution_context,
require_markdown=body.require_markdown,
)
except DuplicateTaskNameError:
raise HTTPException(status_code=409, detail="已存在同名定时任务,请换一个名称")
offline-backend-20260512/backend/app/gateway/routers/skills.py
+2313 -1284
View File
File diff suppressed because it is too large. Load diff
offline-backend-20260512/backend/app/gateway/routers/system_settings.py
+138 -1
View File
@@ -3,10 +3,12 @@
from __future__ import annotations
import logging
from typing import Literal
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.auth.password import hash_password_async
from app.gateway.deps import get_optional_user_from_request
from deerflow.config.system_settings import (
PromptPrefixSettings,
@@ -31,6 +33,30 @@ class PromptPrefixUpdateRequest(BaseModel):
prompt_prefix: str | None = None
class PrivateSquareStatusResponse(BaseModel):
"""Status of the private-square feature, readable by any logged-in user.
The password hash itself is **never** returned — the response only signals
whether the feature is enabled and whether a password has been set.
"""
enabled: bool = Field(..., description="Whether the private square is exposed to users")
password_set: bool = Field(..., description="Whether a password is currently configured")
class PrivateSquareUpdateRequest(BaseModel):
enabled: bool | None = Field(default=None, description="Toggle the private-square feature")
password: str | None = Field(default=None, description="New password in plain text. Pass an empty string to clear.")
class CitationDisplayResponse(BaseModel):
cleanup_words: list[str] = Field(default_factory=list, description="Global citation cleanup words")
class CitationDisplayUpdateRequest(BaseModel):
cleanup_words: list[str] | None = None
def _to_response(settings: PromptPrefixSettings) -> PromptPrefixResponse:
return PromptPrefixResponse(
enabled=settings.enabled,
@@ -61,5 +87,116 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
current = load_system_settings()
fields = body.model_dump(exclude_unset=True)
updated_prefix = current.prompt_prefix.model_copy(update=fields)
saved = save_system_settings(SystemSettings(prompt_prefix=updated_prefix))
saved = save_system_settings(current.model_copy(update={"prompt_prefix": updated_prefix}))
return _to_response(saved.prompt_prefix)
@router.get("/citation-display", response_model=CitationDisplayResponse)
async def get_citation_display() -> CitationDisplayResponse:
"""Return global citation display cleanup settings."""
settings = load_system_settings().citation_display
return CitationDisplayResponse(cleanup_words=settings.cleanup_words)
@router.put("/citation-display", response_model=CitationDisplayResponse)
async def update_citation_display(
body: CitationDisplayUpdateRequest,
request: Request,
) -> CitationDisplayResponse:
"""Update global citation display cleanup settings. Admin only."""
await _require_admin(request)
current = load_system_settings()
words = [
str(item).strip()
for item in (body.cleanup_words or [])
if str(item).strip()
]
updated = current.citation_display.model_copy(update={"cleanup_words": words})
saved = save_system_settings(
current.model_copy(update={"citation_display": updated})
)
return CitationDisplayResponse(cleanup_words=saved.citation_display.cleanup_words)
@router.get("/private-square", response_model=PrivateSquareStatusResponse)
async def get_private_square_status() -> PrivateSquareStatusResponse:
"""Return whether the private square is enabled and a password is set."""
settings = load_system_settings().private_square
return PrivateSquareStatusResponse(
enabled=settings.enabled,
password_set=bool(settings.password_hash),
)
@router.put("/private-square", response_model=PrivateSquareStatusResponse)
async def update_private_square(body: PrivateSquareUpdateRequest, request: Request) -> PrivateSquareStatusResponse:
"""Update the private-square configuration. Admin only.
Setting ``password`` to a non-empty string rehashes and stores it.
Passing an empty string clears the password (effectively locking the
private square so no one can read it).
"""
await _require_admin(request)
current = load_system_settings()
updated = current.private_square
fields = body.model_dump(exclude_unset=True)
update: dict[str, object] = {}
if "enabled" in fields and fields["enabled"] is not None:
update["enabled"] = bool(fields["enabled"])
if "password" in fields and fields["password"] is not None:
plain = str(fields["password"])
if plain == "":
update["password_hash"] = ""
else:
update["password_hash"] = await hash_password_async(plain)
if update:
updated = updated.model_copy(update=update)
saved = save_system_settings(current.model_copy(update={"private_square": updated}))
else:
saved = current
return PrivateSquareStatusResponse(
enabled=saved.private_square.enabled,
password_set=bool(saved.private_square.password_hash),
)
class AppearanceResponse(BaseModel):
"""Effective system-wide appearance defaults."""
default_theme: Literal["light", "dark"] = Field(..., description="System default theme — light or dark")
class AppearanceUpdateRequest(BaseModel):
default_theme: Literal["light", "dark"] | None = Field(default=None)
@router.get("/appearance", response_model=AppearanceResponse)
async def get_appearance_settings() -> AppearanceResponse:
"""Return the global default theme. Any logged-in user may read this."""
settings = load_system_settings().appearance
return AppearanceResponse(default_theme=settings.default_theme)
@router.put("/appearance", response_model=AppearanceResponse)
async def update_appearance_settings(
body: AppearanceUpdateRequest, request: Request
) -> AppearanceResponse:
"""Update the global appearance defaults. Admin only."""
await _require_admin(request)
current = load_system_settings()
fields = body.model_dump(exclude_unset=True)
update: dict[str, object] = {}
if "default_theme" in fields and fields["default_theme"] is not None:
update["default_theme"] = fields["default_theme"]
if update:
updated_appearance = current.appearance.model_copy(update=update)
saved = save_system_settings(
current.model_copy(update={"appearance": updated_appearance})
)
else:
saved = current
return AppearanceResponse(default_theme=saved.appearance.default_theme)
offline-backend-20260512/backend/app/gateway/routers/threads.py
+498 -39
View File
@@ -12,24 +12,38 @@ matching the LangGraph Platform wire format expected by the
from __future__ import annotations
import logging
import uuid
from typing import Any
import logging
import json
import os
from pathlib import Path
import uuid
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from langgraph.checkpoint.base import empty_checkpoint
from pydantic import BaseModel, Field, field_validator
from fastapi import APIRouter, HTTPException, Request
from langgraph.checkpoint.base import empty_checkpoint
from pydantic import BaseModel, Field, field_validator
from sqlalchemy import select
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer
from app.gateway.utils import sanitize_log_param
from deerflow.config.extensions_config import SkillDisplayConfig
from deerflow.config.paths import Paths, get_paths
from deerflow.persistence.engine import get_engine, get_session_factory
from deerflow.persistence.skills.model import SkillDisplayAdapterRow
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.references import (
adapt_reference_batches_with_displays,
build_reference_batches as build_runtime_reference_batches,
build_reference_debug_payload,
build_reference_debug_summary,
)
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer
from app.gateway.utils import sanitize_log_param
from deerflow.config.paths import Paths, get_paths
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["threads"])
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["threads"])
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
# Metadata keys that the server controls; clients are not allowed to set
@@ -41,16 +55,86 @@ router = APIRouter(prefix="/api/threads", tags=["threads"])
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
"""Return ``metadata`` with server-controlled keys removed."""
if not metadata:
return metadata or {}
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
# ---------------------------------------------------------------------------
# Response / request models
# ---------------------------------------------------------------------------
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
"""Return ``metadata`` with server-controlled keys removed."""
if not metadata:
return metadata or {}
return {k: v for k, v in metadata.items() if k not in _SERVER_RESERVED_METADATA_KEYS}
def _get_rag_debug_logger() -> logging.Logger:
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
if getattr(debug_logger, "_rag_file_configured", False):
return debug_logger
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
if not log_path.is_absolute():
log_path = Path.cwd() / log_path
log_path.parent.mkdir(parents=True, exist_ok=True)
handler = logging.FileHandler(log_path, encoding="utf-8")
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
debug_logger.addHandler(handler)
debug_logger.setLevel(logging.INFO)
debug_logger.propagate = False
setattr(debug_logger, "_rag_file_configured", True)
setattr(debug_logger, "_rag_file_path", str(log_path))
return debug_logger
def _rag_debug_file_path() -> str:
return str(getattr(_get_rag_debug_logger(), "_rag_file_path", "logs/rag_citation_debug.log"))
def _log_rag_debug(message: str, *args: Any) -> None:
logger.info(message, *args)
_get_rag_debug_logger().info(message, *args)
def _json_text_loads(value: str | None) -> dict[str, Any] | None:
if not value:
return None
try:
parsed = json.loads(value)
return parsed if isinstance(parsed, dict) else None
except Exception:
logger.debug("Failed to parse persisted skill display JSON", exc_info=True)
return None
async def _load_persisted_skill_displays() -> dict[str, SkillDisplayConfig]:
engine = get_engine()
sf = get_session_factory()
if engine is None or sf is None:
return {}
try:
async with engine.begin() as conn:
await conn.run_sync(SkillDisplayAdapterRow.__table__.create, checkfirst=True)
async with sf() as session:
result = await session.execute(select(SkillDisplayAdapterRow))
displays: dict[str, SkillDisplayConfig] = {}
for row in result.scalars():
payload = _json_text_loads(row.display_config)
if payload is None:
continue
try:
display = SkillDisplayConfig.model_validate(payload)
except Exception:
logger.debug(
"Failed to validate persisted skill display for %s",
row.skill_name,
exc_info=True,
)
continue
if display.enabled and display.mode != "none":
displays[row.skill_name] = display
return displays
except Exception:
logger.debug("Failed to load persisted skill display adapters", exc_info=True)
return {}
# ---------------------------------------------------------------------------
# Response / request models
# ---------------------------------------------------------------------------
class ThreadDeleteResponse(BaseModel):
@@ -132,11 +216,45 @@ class HistoryEntry(BaseModel):
next: list[str] = Field(default_factory=list)
class ThreadHistoryRequest(BaseModel):
class ThreadHistoryRequest(BaseModel):
"""Request body for checkpoint history."""
limit: int = Field(default=10, ge=1, le=100, description="Maximum entries")
before: str | None = Field(default=None, description="Cursor for pagination")
before: str | None = Field(default=None, description="Cursor for pagination")
class ReferenceSourceResponse(BaseModel):
index: int
id: str | None = None
skillName: str = "backend-reference"
mode: str = "citation"
title: str
type: str = "result"
snippet: str = ""
source: str | None = None
author: str | None = None
time: str | None = None
url: str | None = None
content: str | None = None
score: float | None = None
tableColumns: list[dict[str, Any]] = Field(default_factory=list)
raw: dict[str, Any] = Field(default_factory=dict)
class ReferenceBatchResponse(BaseModel):
id: str
assistant_message_id: str
sources: list[ReferenceSourceResponse] = Field(default_factory=list)
class ReferenceBatchesResponse(BaseModel):
batches: list[ReferenceBatchResponse] = Field(default_factory=list)
class ReferenceDebugLogRequest(BaseModel):
event: str
thread_id: str | None = None
payload: dict[str, Any] = Field(default_factory=dict)
# ---------------------------------------------------------------------------
@@ -163,7 +281,7 @@ def _delete_thread_data(thread_id: str, paths: Paths | None = None, *, user_id:
return ThreadDeleteResponse(success=True, message=f"Deleted local thread data for {thread_id}")
def _derive_thread_status(checkpoint_tuple) -> str:
def _derive_thread_status(checkpoint_tuple) -> str:
"""Derive thread status from checkpoint metadata."""
if checkpoint_tuple is None:
return "idle"
@@ -179,7 +297,251 @@ def _derive_thread_status(checkpoint_tuple) -> str:
if tasks:
return "interrupted"
return "idle"
return "idle"
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "")
return str(getattr(message, "type", "") or "")
def _message_id(message: Any) -> str | None:
if isinstance(message, dict):
value = message.get("id")
else:
value = getattr(message, "id", None)
return str(value) if value else None
def _message_content(message: Any) -> Any:
if isinstance(message, dict):
return message.get("content")
return getattr(message, "content", None)
def _message_name(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
return str(getattr(message, "name", None) or "tool")
def _has_tool_calls(message: Any) -> bool:
if isinstance(message, dict):
direct = message.get("tool_calls")
chunks = message.get("tool_call_chunks")
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
else:
direct = getattr(message, "tool_calls", None)
chunks = getattr(message, "tool_call_chunks", None)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
return bool(direct or chunks or extra)
def _has_answer_content(message: Any) -> bool:
content = _message_content(message)
if isinstance(content, str):
return bool(content.strip())
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text" and str(part.get("text") or "").strip():
return True
return False
def _extract_tool_payload(message: Any) -> Any:
content = _message_content(message)
if isinstance(content, dict):
return _unwrap_tool_payload(content)
if isinstance(content, list):
texts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
if isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "json" and isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "text" and isinstance(part.get("text"), str):
texts.append(part["text"])
unwrapped = _unwrap_tool_payload(part)
if unwrapped is not None:
return unwrapped
content = "\n".join(texts)
if isinstance(content, str):
return _parse_json_like(content)
return None
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
if any(isinstance(value.get(key), list) for key in ("results", "data", "items", "hits", "documents", "records", "sources")):
return value
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
nested = value.get(key)
if isinstance(nested, str):
parsed = _parse_json_like(nested)
if parsed is not None:
return parsed
if isinstance(nested, dict):
unwrapped = _unwrap_tool_payload(nested)
if unwrapped is not None:
return unwrapped
return value
def _parse_json_like(text: str) -> Any:
trimmed = text.strip()
if not trimmed:
return None
try:
return json.loads(trimmed)
except Exception:
pass
first_obj = trimmed.find("{")
last_obj = trimmed.rfind("}")
if 0 <= first_obj < last_obj:
try:
return json.loads(trimmed[first_obj : last_obj + 1])
except Exception:
return None
return None
def _first_list(payload: Any) -> list[Any]:
if isinstance(payload, list):
return payload
if not isinstance(payload, dict):
return []
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
value = payload.get(key)
if isinstance(value, list):
return value
for key in ("result", "payload", "response", "output", "data"):
nested = payload.get(key)
found = _first_list(nested)
if found:
return found
return []
def _nested_records(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, dict):
return []
layers = [raw]
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
nested = raw.get(key)
if isinstance(nested, dict):
layers.append(nested)
return layers
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
for layer in layers:
for key in keys:
value = layer.get(key)
if isinstance(value, (str, int, float)) and str(value).strip():
return str(value).strip()
return None
def _normalize_reference_item(raw: Any, fallback_index: int) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
layers = _nested_records(raw)
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
score_raw = _pick(layers, ("score", "relevance", "similarity"))
score = None
if score_raw is not None:
try:
score = float(score_raw)
except Exception:
score = None
if not title:
title = source or f"结果 {fallback_index}"
if not content:
content = ""
if not title and not content and not url:
return None
return {
"id": item_id or url,
"title": title,
"type": "result",
"snippet": content,
"content": content,
"source": source,
"time": time,
"url": url,
"score": score,
"raw": raw,
}
def _reference_key(source: dict[str, Any]) -> str:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
item_id = (
source.get("id")
or source.get("url")
or raw.get("recUuid")
or raw.get("uuid")
or raw.get("docId")
or raw.get("id")
)
if item_id:
return str(item_id).casefold().strip()
return "|".join(
str(source.get(key) or "").casefold().strip()
for key in ("title", "source", "time", "content", "snippet")
)
def _build_reference_batches(messages: list[Any]) -> list[ReferenceBatchResponse]:
batches: list[ReferenceBatchResponse] = []
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
assistant_message: Any | None = None
for cursor in range(end - 1, index, -1):
candidate = messages[cursor]
if (
_message_type(candidate) == "ai"
and _message_id(candidate)
and _has_answer_content(candidate)
and not _has_tool_calls(candidate)
):
assistant_message = candidate
break
if assistant_message is None:
continue
seen: set[str] = set()
sources: list[ReferenceSourceResponse] = []
for tool_message in messages[index + 1 : end]:
if _message_type(tool_message) != "tool":
continue
payload = _extract_tool_payload(tool_message)
for raw_item in _first_list(payload):
normalized = _normalize_reference_item(raw_item, len(sources) + 1)
if not normalized:
continue
key = _reference_key(normalized)
if key and key in seen:
continue
if key:
seen.add(key)
sources.append(ReferenceSourceResponse(index=len(sources) + 1, **normalized))
if sources:
mid = _message_id(assistant_message) or str(index)
batches.append(ReferenceBatchResponse(id=mid, assistant_message_id=mid, sources=sources))
return batches
# ---------------------------------------------------------------------------
@@ -406,9 +768,9 @@ async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
# ---------------------------------------------------------------------------
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
"""Get the latest state snapshot for a thread.
Channel values are serialized to ensure LangChain message objects
@@ -455,12 +817,109 @@ async def get_thread_state(thread_id: str, request: Request) -> ThreadStateRespo
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
)
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_reference_batches(thread_id: str, request: Request) -> ReferenceBatchesResponse:
"""Return backend-built citation batches for completed assistant turns."""
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get reference batches for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get reference batches")
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
messages = values.get("messages")
if not isinstance(messages, list):
_log_rag_debug(
"backend reference-batches thread=%s messages=missing batches=0 log_file=%s",
sanitize_log_param(thread_id),
_rag_debug_file_path(),
)
return ReferenceBatchesResponse(batches=[])
batches_before_display_adapt = build_runtime_reference_batches(messages)
raw_batches = batches_before_display_adapt
persisted_displays = await _load_persisted_skill_displays()
if persisted_displays:
raw_batches = adapt_reference_batches_with_displays(raw_batches, persisted_displays)
debug_summary = build_reference_debug_summary(messages, raw_batches)
_log_rag_debug(
"backend reference-batches thread=%s messages=%s turns=%s tool_messages=%s raw_items=%s "
"normalized_items=%s kept_items=%s batches=%s batch_sources=%s batch_ids=%s "
"display_adapters=%s turns_detail=%s log_file=%s",
sanitize_log_param(thread_id),
debug_summary["messages"],
debug_summary["turns"],
debug_summary["tool_messages"],
debug_summary["raw_items"],
debug_summary["normalized_items"],
debug_summary["kept_items"],
len(raw_batches),
",".join(str(batch.get("source_count", 0)) for batch in debug_summary["batches"]),
",".join(str(batch.get("id") or "") for batch in debug_summary["batches"]),
len(persisted_displays),
json.dumps(debug_summary["turns_detail"], ensure_ascii=False),
_rag_debug_file_path(),
)
_log_rag_debug(
"backend reference-batches-payload thread=%s %s",
sanitize_log_param(thread_id),
json.dumps(
{
"display_adapter_names": sorted(persisted_displays.keys()),
"before_display_adapt": build_reference_debug_payload(
messages=messages,
batches=batches_before_display_adapt,
),
"after_display_adapt": build_reference_debug_payload(
messages=messages,
batches=raw_batches,
),
},
ensure_ascii=False,
default=str,
),
)
return ReferenceBatchesResponse(
batches=[
ReferenceBatchResponse(
id=str(batch.get("id") or ""),
assistant_message_id=str(batch.get("assistant_message_id") or batch.get("id") or ""),
sources=[
ReferenceSourceResponse(**source)
for source in batch.get("sources", [])
if isinstance(source, dict)
],
)
for batch in raw_batches
]
)
@router.post("/reference-debug-log")
async def write_reference_debug_log(body: ReferenceDebugLogRequest) -> dict[str, str | bool]:
"""Append frontend citation diagnostics to the backend RAG debug log."""
_get_rag_debug_logger().info(
"frontend %s thread=%s payload=%s log_file=%s",
body.event,
sanitize_log_param(body.thread_id or ""),
json.dumps(body.payload, ensure_ascii=False, default=str),
_rag_debug_file_path(),
)
return {"ok": True, "log_file": _rag_debug_file_path()}
@router.post("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
"""Update thread state (e.g. for human-in-the-loop resume or title rename).
Writes a new checkpoint that merges *body.values* into the latest
offline-backend-20260512/backend/app/gateway/routers/user_prompts.py
+108
View File
@@ -0,0 +1,108 @@
"""CRUD API for a user's personal custom prompts (常用提示词).
Prompts are bound to the requesting user (not stored in the browser), so they
follow the account across devices/sessions.
"""
from __future__ import annotations
import logging
from datetime import datetime
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/user-prompts", tags=["user-prompts"])
class UserPromptResponse(BaseModel):
id: str
title: str = ""
content: str
sort_order: int = 0
created_at: datetime | str | None = None
updated_at: datetime | str | None = None
class UserPromptsListResponse(BaseModel):
prompts: list[UserPromptResponse]
class UserPromptCreateRequest(BaseModel):
title: str = Field(default="", max_length=255)
content: str = Field(..., min_length=1)
sort_order: int = Field(default=0)
class UserPromptUpdateRequest(BaseModel):
title: str | None = Field(default=None, max_length=255)
content: str | None = Field(default=None, min_length=1)
sort_order: int | None = None
def _current_user_id(request: Request) -> str:
user = getattr(request.state, "user", None)
if user is not None:
return str(user.id)
return get_effective_user_id()
def _get_store(request: Request):
store = getattr(request.app.state, "user_prompt_store", None)
if store is None:
raise HTTPException(status_code=503, detail="User prompt store not available")
return store
@router.get("", response_model=UserPromptsListResponse)
async def list_user_prompts(request: Request) -> UserPromptsListResponse:
store = _get_store(request)
user_id = _current_user_id(request)
rows = await store.list_for_user(user_id)
return UserPromptsListResponse(prompts=[UserPromptResponse(**r) for r in rows])
@router.post("", response_model=UserPromptResponse, status_code=201)
async def create_user_prompt(request: Request, body: UserPromptCreateRequest) -> UserPromptResponse:
store = _get_store(request)
user_id = _current_user_id(request)
row = await store.create(
user_id,
{
"id": uuid4().hex,
"title": (body.title or "").strip(),
"content": body.content,
"sort_order": body.sort_order,
},
)
return UserPromptResponse(**row)
@router.put("/{prompt_id}", response_model=UserPromptResponse)
async def update_user_prompt(request: Request, prompt_id: str, body: UserPromptUpdateRequest) -> UserPromptResponse:
store = _get_store(request)
user_id = _current_user_id(request)
data: dict = {}
if body.title is not None:
data["title"] = body.title.strip()
if body.content is not None:
data["content"] = body.content
if body.sort_order is not None:
data["sort_order"] = body.sort_order
row = await store.update(prompt_id, user_id, data)
if row is None:
raise HTTPException(status_code=404, detail="Prompt not found")
return UserPromptResponse(**row)
@router.delete("/{prompt_id}", status_code=204)
async def delete_user_prompt(request: Request, prompt_id: str) -> None:
store = _get_store(request)
user_id = _current_user_id(request)
deleted = await store.delete(prompt_id, user_id)
if not deleted:
raise HTTPException(status_code=404, detail="Prompt not found")
offline-backend-20260512/backend/app/gateway/services.py
+183 -6
View File
@@ -11,6 +11,7 @@ import asyncio
import json
import logging
import re
import time
from collections.abc import Mapping
from typing import Any
@@ -102,6 +103,14 @@ _DEFAULT_ASSISTANT_ID = "lead_agent"
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent", "ai_writing"})
_RAG_CITATION_PROMPT = """当前对话已开启参考文献模式。
如果本轮调用了搜索、知识库、检索类技能或工具，请基于本轮所有检索结果整体回答。
如果你已经知道需要多个检索词，请优先在同一步里并行调用检索工具，等这些检索都返回后再回答，不要查一个检索词就先写一版答案。
你必须把同一轮内多次检索得到的资料视为同一个 referenceBatch，按系统稍后提供的合并批次从 [1] 开始连续编号，不要让每个检索词各自从 [1] 重新编号。
回答正文中的关键结论必须标注对应编号，例如 [1]、[2]，多个来源可写为 [1][3][5]。
不要引用不存在的编号，不要使用技能内部自带的编号，不要把参考文献完整列表堆在正文末尾。
如果没有任何可用检索结果，请直接说明未检索到可靠参考资料。"""
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
"""Return ``graph_input`` with the prefix *tagged* onto its first HumanMessage.
@@ -140,6 +149,43 @@ def _prompt_prefix_opted_in(body: Any) -> bool:
return False
def _rag_mode_requested(body: Any) -> bool:
context = getattr(body, "context", None)
if isinstance(context, Mapping) and context.get("rag_mode_enabled"):
return True
metadata = getattr(body, "metadata", None)
if isinstance(metadata, Mapping) and (
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
):
return True
return False
def _thread_rag_mode_enabled(record: Any) -> bool:
if not isinstance(record, Mapping):
return False
metadata = record.get("metadata")
return isinstance(metadata, Mapping) and bool(
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
)
def _append_invisible_prompt(graph_input: dict[str, Any], prompt: str) -> dict[str, Any]:
messages = graph_input.get("messages")
if not isinstance(messages, list) or not messages:
return graph_input
new_messages = list(messages)
for idx, msg in enumerate(new_messages):
if not isinstance(msg, HumanMessage):
continue
extra = dict(getattr(msg, "additional_kwargs", None) or {})
existing = str(extra.get("prompt_prefix") or "").strip()
extra["prompt_prefix"] = f"{existing}\n\n{prompt}".strip() if existing else prompt
new_messages[idx] = msg.model_copy(update={"additional_kwargs": extra})
break
return {**graph_input, "messages": new_messages}
# Whitelist of run-context keys that the langgraph-compat layer forwards from
# ``body.context`` into the run config. ``config["context"]`` exists in
# LangGraph >=0.6, but these values must be written to both ``configurable``
@@ -153,6 +199,7 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"mode",
"thinking_enabled",
"reasoning_effort",
"rag_mode_enabled",
"writing_mode",
"writing_artifact_path",
"canvas_artifact_title",
@@ -316,8 +363,17 @@ def _collect_requested_agent_ids(body: Any) -> set[str]:
async def _sync_legacy_agent_records(store) -> None:
"""Import existing folder-backed agents as built-in records."""
import sys
"""Import existing folder-backed agents as built-in records.
Heavy: ``list_custom_agents()`` scans ``.deer-flow/agents/`` and reads
``config.yaml`` + ``SOUL.md`` per directory, then ``store.ensure_builtin``
does one DB round-trip per agent. On a machine with N agent dirs this is
O(N) file IO + O(N) DB calls — historically the worst per-request hotspot
in ``start_run`` (observed 30s+ on machines with 100+ residual coordinator
shells + slow MySQL). **Never call this directly from request handlers** —
use :func:`_maybe_sync_legacy_agent_records` which applies the same TTL
gating as ``routers/agents._maybe_sync_legacy_agents``.
"""
from deerflow.config.agents_config import list_custom_agents
print(f"[_sync_legacy_agent_records] store type: {type(store).__name__}, id: {id(store)}", file=sys.stderr)
@@ -338,6 +394,69 @@ async def _sync_legacy_agent_records(store) -> None:
logger.info("_sync_legacy_agent_records: done")
# Throttle per-request legacy-sync. Matches ``routers/agents._LEGACY_SYNC_TTL_SECONDS``
# semantics — the real sync runs once at startup (lifespan); this is a safety
# net for environments that bypass the lifespan hook or for freshly dropped
# agent directories. Lifespan registers itself via the same state by calling
# the agents router's ``_mark_legacy_sync_fresh()`` after its boot-time sync,
# so this module's TTL state is independent (a separate gate kept here so we
# don't import from ``routers/agents`` and create a circular import).
_LEGACY_SYNC_TTL_SECONDS = 3600.0
_legacy_sync_state: dict[str, Any] = {"last_at": 0.0, "lock": None}
# Anything slower than this triggers the WARNING-level breakdown in
# ``start_run``. 500ms gives plenty of headroom for cold caches + a few DB
# round-trips on healthy systems while still catching the pathological
# multi-second cases that brought us here.
_START_RUN_SLOW_THRESHOLD_SECONDS = 0.5
def _mark_legacy_sync_fresh() -> None:
"""Tell this module's TTL gate that a sync just completed elsewhere.
Called by the Gateway lifespan hook after it runs the boot-time legacy
agent sync (``routers/agents._sync_legacy_agents``). Without this, the
first request after startup would race past the TTL gate here and trigger
a redundant rescan even though the data is already fresh.
"""
_legacy_sync_state["last_at"] = time.monotonic()
async def _maybe_sync_legacy_agent_records(store) -> None:
"""TTL-gated wrapper around :func:`_sync_legacy_agent_records`.
Steady-state behavior: returns in microseconds inside the TTL window — no
filesystem walk, no DB round-trips. Outside the TTL window we serialize
callers under an :class:`asyncio.Lock` so a burst of concurrent
``/runs/stream`` requests fans into a single rescan instead of N parallel
fan-outs. Pattern mirrors ``routers/agents._maybe_sync_legacy_agents``.
"""
now = time.monotonic()
if now - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
return
lock = _legacy_sync_state["lock"]
if lock is None:
lock = asyncio.Lock()
_legacy_sync_state["lock"] = lock
async with lock:
# Re-check under the lock — a sibling coroutine may have just refreshed.
if time.monotonic() - float(_legacy_sync_state["last_at"]) < _LEGACY_SYNC_TTL_SECONDS:
return
t_sync_start = time.perf_counter()
await _sync_legacy_agent_records(store)
elapsed = time.perf_counter() - t_sync_start
_legacy_sync_state["last_at"] = time.monotonic()
# When this fires it's a "first request of the hour" warmup — log it
# so operators can see the spike isn't an isolated bug. Anything
# over 2s usually means a lot of agent directories + slow IO/DB.
logger.info(
"[start-run-timing] legacy-agent sync warmed cache in %.3fs (next sync in %.0fs)",
elapsed,
_LEGACY_SYNC_TTL_SECONDS,
)
async def _enforce_agent_access(body: Any, request: Request) -> None:
agent_ids = _collect_requested_agent_ids(body)
if not agent_ids:
@@ -348,9 +467,7 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
from deerflow.runtime.user_context import get_effective_user_id
store = get_agent_store(request)
import sys
print(f"[_enforce_agent_access] store type: {type(store).__name__}, id: {id(store)}", file=sys.stderr)
await _sync_legacy_agent_records(store)
await _maybe_sync_legacy_agent_records(store)
user_id = get_effective_user_id()
logger.info(f"_enforce_agent_access: user_id={user_id}, agent_ids={agent_ids}")
@@ -396,14 +513,27 @@ async def start_run(
Target thread.
request : Request
FastAPI request — used to retrieve singletons from ``app.state``.
Timing observability
--------------------
Each phase records ``perf_counter`` deltas. When the total wallclock from
function entry to ``run_agent`` task creation exceeds
``_START_RUN_SLOW_THRESHOLD_SECONDS`` we emit a ``[start-run-timing]``
breakdown at WARNING level. Stays at DEBUG when fast, so steady-state
logs are not polluted. Investigate spikes by grep'ing
``[start-run-timing]`` together with ``[upstream-timing]`` (multi-agent
router) and ``[leader-timing] / [special-timing]`` (圆桌 leader/special).
"""
t_entry = time.perf_counter()
bridge = get_stream_bridge(request)
run_mgr = get_run_manager(request)
run_ctx = get_run_context(request)
disconnect = DisconnectMode.cancel if body.on_disconnect == "cancel" else DisconnectMode.continue_
t_pre_enforce = time.perf_counter()
await _enforce_agent_access(body, request)
t_post_enforce = time.perf_counter()
try:
record = await run_mgr.create_or_reject(
@@ -418,20 +548,34 @@ async def start_run(
raise HTTPException(status_code=409, detail=str(exc)) from exc
except UnsupportedStrategyError as exc:
raise HTTPException(status_code=501, detail=str(exc)) from exc
t_post_create = time.perf_counter()
# Upsert thread metadata so the thread appears in /threads/search,
# even for threads that were never explicitly created via POST /threads
# (e.g. stateless runs).
requested_rag_mode = _rag_mode_requested(body)
effective_rag_mode = requested_rag_mode
try:
existing = await run_ctx.thread_store.get(thread_id)
effective_rag_mode = requested_rag_mode or _thread_rag_mode_enabled(existing)
if existing is None:
metadata = dict(body.metadata or {})
if effective_rag_mode:
metadata["rag_mode_enabled"] = True
metadata["rag_mode"] = True
await run_ctx.thread_store.create(
thread_id,
assistant_id=body.assistant_id,
metadata=body.metadata,
metadata=metadata,
)
else:
await run_ctx.thread_store.update_status(thread_id, "running")
if requested_rag_mode and not _thread_rag_mode_enabled(existing):
await run_ctx.thread_store.update_metadata(
thread_id,
{"rag_mode_enabled": True, "rag_mode": True},
)
effective_rag_mode = True
# SDK 的 ``client.threads.create()`` 不传 assistant_id —— 先建出来空的 thread，
# 真正提交 run 时才知道用哪个 assistant。这里回填一次，避免历史列表 / 按
# assistant 筛选时拿不到关联。
@@ -439,6 +583,7 @@ async def start_run(
await run_ctx.thread_store.update_assistant_id(thread_id, body.assistant_id)
except Exception:
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
t_post_thread_upsert = time.perf_counter()
agent_factory = resolve_agent_factory(body.assistant_id)
graph_input = normalize_input(body.input)
@@ -455,6 +600,8 @@ async def start_run(
prefix_settings = load_system_settings().prompt_prefix
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
if effective_rag_mode:
graph_input = _append_invisible_prompt(graph_input, _RAG_CITATION_PROMPT)
config = build_run_config(thread_id, body.config, body.metadata, assistant_id=body.assistant_id)
# Merge DeerFlow-specific context overrides into both ``configurable`` and ``context``.
@@ -462,6 +609,8 @@ async def start_run(
# that carries agent configuration (model_name, thinking_enabled, etc.).
# Only agent-relevant keys are forwarded; unknown keys (e.g. thread_id) are ignored.
body_context = dict(getattr(body, "context", None) or {})
if effective_rag_mode:
body_context["rag_mode_enabled"] = True
# Promote whitelisted top-level body fields into the context map so
# callers can pass them either nested under ``context`` or as flat
# top-level fields (``excluded_tools``, ``skill_stop_names``, …).
@@ -473,6 +622,7 @@ async def start_run(
stream_modes = normalize_stream_modes(body.stream_mode)
t_pre_task = time.perf_counter()
task = asyncio.create_task(
run_agent(
bridge,
@@ -497,6 +647,33 @@ async def start_run(
# title from the checkpoint and calls thread_store.update_display_name
# after the run completes.
total = t_pre_task - t_entry
if total >= _START_RUN_SLOW_THRESHOLD_SECONDS:
# Slow path: emit a phase breakdown so on-call can identify the
# culprit without re-running with extra instrumentation. The most
# common cause is a stale ``_maybe_sync_legacy_agent_records`` TTL
# window firing on a machine with hundreds of agent directories.
logger.warning(
"[start-run-timing] thread=%s total=%.3fs enforce_access=%.3fs "
"run_create=%.3fs thread_upsert=%.3fs build_payload=%.3fs",
sanitize_log_param(thread_id),
total,
t_post_enforce - t_pre_enforce,
t_post_create - t_post_enforce,
t_post_thread_upsert - t_post_create,
t_pre_task - t_post_thread_upsert,
)
else:
logger.debug(
"[start-run-timing] thread=%s total=%.3fs enforce=%.3fs create=%.3fs upsert=%.3fs build=%.3fs",
sanitize_log_param(thread_id),
total,
t_post_enforce - t_pre_enforce,
t_post_create - t_post_enforce,
t_post_thread_upsert - t_post_create,
t_pre_task - t_post_thread_upsert,
)
return record
offline-backend-20260512/backend/config.yaml
+23 -15
View File
@@ -10,11 +10,24 @@ gateway:
port: 8001
models:
- name: zai-org/GLM-5-FP8
display_name: zai-org/GLM-5-FP8
use: langchain_openai:ChatOpenAI
model: zai-org/GLM-5-FP8
api_key: Bearer modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
base_url: https://api.us-west-2.modal.direct/v1
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: deepseek-chat
display_name: DeepSeek Chat
use: langchain_openai:ChatOpenAI
model: deepseek-chat
api_key: sk-e1c22b7931dc446f8b67e57878edd105
api_key: sk-1520028668cb4cc388ed8315ff1e856e
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
@@ -58,16 +71,6 @@ models:
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: minimax-m2.5
display_name: MiniMax M2.5
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.5
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com/v1
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: false
title:
enabled: true
@@ -96,6 +99,11 @@ tools:
use: deerflow.community.ddg_search.tools:web_search_tool
max_results: 5
- name: web_fetch
group: web
use: deerflow.community.jina_ai.tools:web_fetch_tool
timeout: 15
- name: ls
group: file:read
use: deerflow.sandbox.tools:ls_tool
@@ -131,7 +139,7 @@ sandbox:
allow_host_bash: true
skills:
path: D:\project\bj\deerflow-server\skills
path: skills
security_scan:
enabled: false
@@ -139,14 +147,14 @@ agents_api:
enabled: true
database:
backend: mysql
mysql_url: $MYSQL_DATABASE_URL
backend: sqlite
sqlite_dir: .deer-flow/data
pool_size: 10
echo_sql: false
checkpointer:
type: sqlite
connection_string: D:\cursor-work\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
connection_string: D:\05271502\deerflow-server\offline-backend-20260512\backend\.deer-flow\data\deerflow.db
subagents:
enabled: true
offline-backend-20260512/backend/extensions_config.json
+11
View File
@@ -0,0 +1,11 @@
{
"mcpServers": {},
"skills": {
"web-research": { "enabled": true },
"deep-search": { "enabled": true },
"code-retrieval": { "enabled": true },
"news-briefing": { "enabled": true },
"workspace-search": { "enabled": true },
"policy-knowledge": { "enabled": true }
}
}
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/graph.py
+16 -3
View File
@@ -55,6 +55,9 @@ async def pause_material_confirm(state: AIWritingState) -> dict:
action = user_input.get("action", "confirm")
# 单输入框文本：可能是技能名 / 来源 / 检索意图，re_search 和 enable_skill_search 都用得到
user_query = (user_input.get("userQuery") or user_input.get("user_query") or "").strip()
if action == "confirm":
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
@@ -66,12 +69,22 @@ async def pause_material_confirm(state: AIWritingState) -> dict:
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材，开始规划大纲"}],
}
# re_search：重新搜索
# enable_skill_search：用户选择"启用技能检索"，回到 researcher 走 skill 模式
if action == "enable_skill_search":
msg = f"已启用技能检索（输入：{user_query[:30]}）" if user_query else "已启用技能检索，正在筛选可用技能…"
return {
"last_user_intervention": {**user_input, "action": "enable_skill_search", "user_query": user_query},
"status": "researching",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": msg}],
}
# re_search：重新搜索（支持新版 user_query 单输入框 + 旧版 extraKeywords 数组）
extra_kws = user_input.get("extraKeywords") or []
msg = f"按输入「{user_query[:30]}」重新搜索…" if user_query else "重新搜索中..."
return {
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws, "user_query": user_query},
"status": "researching",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": msg}],
}
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/nodes/intent_parser.py
+64 -20
View File
@@ -83,11 +83,16 @@ _SYSTEM_TEMPLATE = """你是一名写作助手。
_USER_TEMPLATE = """用户写作意图：{user_intent}
请分析并输出以下 JSON（如果意图中未明确某字段，给出合理默认值）：
用户在表单中已指定写作配置（以下三项必须原样输出，不要自行修改）：
- 文章类型（article_type）：{preconfigured_article_type}
- 目标读者（target_audience）：{preconfigured_audience}
- 目标字数（word_count_target）：{preconfigured_word_count}
请输出以下 JSON，article_type / target_audience / word_count_target 必须与上述一致，只需根据用户意图精炼 refined_intent：
{{{{
"article_type": "{type_options} 之一",
"target_audience": "目标读者描述",
"word_count_target": 800,
"article_type": "{preconfigured_article_type}",
"target_audience": "{preconfigured_audience}",
"word_count_target": {preconfigured_word_count},
"refined_intent": "精炼后的写作意图（一句话）"
}}}}"""
@@ -124,6 +129,44 @@ def _coerce_word_count(raw) -> int | None:
return value if value > 0 else None
def _resolve_intent_fields(
state: AIWritingState,
result: dict,
fallback_result: dict,
valid_keys: list[str],
default_key: str,
) -> tuple[str, str, int, str]:
"""合并 LLM 解析结果与用户表单预填字段；预填字段优先（见 AI_WRITING_BACKEND_DEV_ZH §intent_parser）。"""
fallback_type = fallback_result["article_type"]
if fallback_type not in valid_keys:
fallback_type = default_key
raw_type = result.get("article_type", "")
llm_type = raw_type if raw_type in valid_keys else fallback_type
preconfigured_type = (state.get("article_type") or "").strip()
if preconfigured_type in valid_keys:
final_article_type = preconfigured_type
else:
final_article_type = llm_type
preconfigured_audience = (state.get("target_audience") or "").strip()
if preconfigured_audience:
final_target_audience = preconfigured_audience
else:
final_target_audience = result.get("target_audience") or fallback_result["target_audience"]
preconfigured_wc = state.get("word_count_target")
if isinstance(preconfigured_wc, int) and preconfigured_wc > 0:
final_word_count = preconfigured_wc
else:
word_count = _coerce_word_count(result.get("word_count_target"))
final_word_count = word_count if word_count is not None else fallback_result["word_count_target"]
final_refined_intent = result.get("refined_intent") or state["user_intent"]
return final_article_type, final_target_audience, final_word_count, final_refined_intent
async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> dict:
# 从 configurable 读取动态类型列表，未配置时退回默认值
configurable = (config or {}).get("configurable", {})
@@ -137,10 +180,6 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
type_options = _build_type_options(valid_article_types)
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
user = _USER_TEMPLATE.format(
user_intent=state["user_intent"],
type_options=" | ".join(valid_keys),
)
fallback_type = state.get("article_type", default_key)
if fallback_type not in valid_keys:
@@ -153,6 +192,17 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
"refined_intent": state["user_intent"],
}
preconfigured_wc = fallback_result["word_count_target"]
if not isinstance(preconfigured_wc, int) or preconfigured_wc <= 0:
preconfigured_wc = 800
user = _USER_TEMPLATE.format(
user_intent=state["user_intent"],
preconfigured_article_type=fallback_type,
preconfigured_audience=fallback_result["target_audience"],
preconfigured_word_count=preconfigured_wc,
)
# 流式调用：与 writer_outline / writer_draft 同套机制 —— LLM 逐字吐出 JSON，
# 通过 ``intent_chunk`` 推给前端打字机；完成后 ``parse_json`` 解析结构化结果，
# 再通过 ``intent_ready`` 推一份结构化字段给前端结果卡片。
@@ -193,13 +243,9 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
"detail": f"意图解析失败，已按默认配置兜底。原因：{llm_error}",
})
# 校验 LLM 返回的 article_type 是否在有效范围内，不合法时回退到默认值
raw_type = result.get("article_type", "")
final_article_type = raw_type if raw_type in valid_keys else fallback_type
final_target_audience = result.get("target_audience") or fallback_result["target_audience"]
word_count = _coerce_word_count(result.get("word_count_target"))
final_word_count = word_count if word_count is not None else fallback_result["word_count_target"]
final_refined_intent = result.get("refined_intent") or state["user_intent"]
final_article_type, final_target_audience, final_word_count, final_refined_intent = _resolve_intent_fields(
state, result, fallback_result, valid_keys, default_key
)
# 推送结构化结果给前端 —— 结果卡片显示文章类型 / 目标读者 / 字数 / 精炼意图
writer({
@@ -214,11 +260,9 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
updates: dict = {"status": "researching", "progress_events": progress_events}
updates["article_type"] = final_article_type
if result.get("target_audience"):
updates["target_audience"] = final_target_audience
if word_count is not None:
updates["word_count_target"] = final_word_count
if result.get("refined_intent"):
updates["target_audience"] = final_target_audience
updates["word_count_target"] = final_word_count
if final_refined_intent != state["user_intent"]:
updates["user_intent"] = final_refined_intent
return updates
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/nodes/researcher.py
+197 -73
View File
@@ -10,6 +10,7 @@ from typing import List
from langchain_core.runnables import RunnableConfig
from ..search import get_search_provider, SearchResult
from ..search.skill_search import SkillPicker
from ..state import AIWritingState, Material, MaterialPackage
from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
@@ -96,15 +97,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 检查干预模式（PAUSE-1 素材确认 / 素材不足求助）
intervention = state.get("last_user_intervention") or {}
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
# PAUSE-1 单输入框：用户写的自然语言（技能名 / 来源 / 检索意图），优先级最高
user_query: str = (intervention.get("user_query") or "").strip()
action = intervention.get("action")
# 素材不足求助点触发的「补充检索」：保留原大纲，检索完直接回写作、不走素材确认/大纲
is_supplement = action == "supplement_search"
# PAUSE-1 素材确认卡用户点「追加检索」并填了新关键词：
# PAUSE-1 素材确认卡用户点「按输入重新搜索」并填了 user_query：
# - 跳过 LLM 重新生成检索词（节省一次调用 + 尊重用户意图）
# - 只检索 extra_keywords 这几个新词，不重检索之前 LLM 出过的词
# - 直接用 user_query 作为唯一检索词
# - 保留 existing material_package，按 URL 去重合并
is_append_only = action == "re_search" and bool(extra_keywords)
is_user_query_search = action == "re_search" and bool(user_query)
# 兼容旧版"追加关键词"调用方（如果还有 client 传 extra_keywords 数组而非 user_query）
is_append_only = action == "re_search" and not user_query and bool(extra_keywords)
# PAUSE-1 素材确认卡用户点「启用技能检索」：
# - 不再走 web/intranet 检索，改用 SkillSearchProvider
# - 顺序调用候选技能 + 早停（第一个技能够就停，不够再加下一个）
# - 旧 material_package 完全丢弃（覆盖式替换，让用户对比 web vs skill 结果）
is_skill_search = action == "enable_skill_search"
# 用户指定的检索关键词数量（1~10），决定生成多少个检索词、检索多少个方向
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
@@ -126,10 +136,17 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
writer({"type": "research_progress", "message": message})
# Step 1: 提取关键词
kw_result: dict = {} # 走追加分支时跳过 LLM 调用，但下方 notice 拼接需要兜底值
if is_append_only:
# 追加模式：直接用用户填的关键词，不再调 LLM
keywords: list[str] = _clean_keywords(extra_keywords, intent)
kw_result: dict = {} # 走"用户输入 / 追加 / 技能"分支时跳过 LLM 调用，notice 拼接需要兜底值
if is_skill_search:
# 技能模式：每个 skill 会自己拼 query（user_query + skill.description），这里不用 LLM 生成关键词
keywords: list[str] = [user_query or intent]
elif is_user_query_search:
# 用户输入分支：把整段自然语言作为唯一检索词，尊重原意，不再做关键词拆分
keywords = [user_query]
_emit(f"按你的输入「{user_query[:30]}」检索…")
elif is_append_only:
# 兼容旧版追加模式：直接用用户填的关键词，不再调 LLM
keywords = _clean_keywords(extra_keywords, intent)
_emit(f"追加检索 {len(keywords)} 个关键词…")
else:
_emit("正在生成检索词…")
@@ -150,78 +167,172 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
# Step 2: 并发搜索
provider = get_search_provider(ai_writing_config)
# Step 2: 搜索
# 技能检索：技能不是素材，是"检索策略 / 视角"。流程：
# a. SkillPicker 选出按相关度排序的 skills（推 list_skills + select_skills 事件）
# b. 按顺序遍历 skills：每个 skill 用其 description 让 LLM 生成针对性关键词
# c. 调真实 web/intranet provider 执行检索，推 invoke_skill + skill_result 事件
# d. 累积素材到 max_materials 即 break，推 stop_early
skill_steps: list[dict] = []
search_errors: list[str] = []
# 按用户指定的关键词数量截断。
# 追加模式：keywords 本身就是用户填的，全部检索不截断（用户填 N 个就搜 N 个）。
# 其他模式：受 keyword_count 限制，但 extra_keywords 在内时上限放宽到 keywords 总数。
if is_append_only:
search_keywords = keywords
def _emit_skill_step(step: dict) -> None:
skill_steps.append(step)
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
if is_skill_search:
# ── 技能编排检索 ──
# 技能选择走流式 LLM（推 skill_select_chunk）+ <think> 分离，前端打字机展示
picker = SkillPicker(
config=config,
model_name=keyword_model,
emit_step=_emit_skill_step,
max_select=5,
user_query=user_query,
)
selected = await picker.pick(user_intent=intent)
search_errors.extend(picker.errors)
# 真实 web/intranet provider（按 config.yaml 配置）
web_provider = get_search_provider(ai_writing_config)
# 每个 skill 尽量多取（不分摊），第一个就达到 max_materials 即可早停；
# 这才贴近"第一个技能够了就停，否则下一个补充"的语义。
per_skill = max(1, max_materials)
raw_materials_pre: list[Material] = []
seen_urls_pre: set[str] = set()
stopped_early = False
for entry in selected:
sk = entry["skill"]
# 用 LLM 把 user_query + skill.description 转成针对性关键词
_emit_skill_step({
"type": "invoke_skill",
"message": f"使用技能「{sk.name}」开始检索…",
"skill_name": sk.name,
})
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
try:
res = await web_provider.search(skill_query, max_results=per_skill)
except Exception as e:
logger.warning("skill[%s] search failed: %s", sk.name, e)
search_errors.append(f"skill[{sk.name}]: {e}")
res = []
# 去重累积
new_count = 0
for r in res or []:
if r.url and r.url in seen_urls_pre:
continue
if r.url:
seen_urls_pre.add(r.url)
raw_materials_pre.append(Material(
id=f"m{len(raw_materials_pre)+1:03d}",
title=r.title,
content=r.content[:500],
source=f"{r.source}（via 技能 {sk.name}）",
published_at=r.published_at,
relevance_score=r.relevance_score,
source_type=r.source_type,
url=r.url,
recUuid=r.recUuid,
))
new_count += 1
_emit_skill_step({
"type": "skill_result",
"message": f"技能「{sk.name}」检索到 {new_count} 条新素材",
"skill_name": sk.name,
"matched_count": new_count,
})
# 早停：累积素材已达上限
if len(raw_materials_pre) >= max_materials:
stopped_early = True
_emit_skill_step({
"type": "stop_early",
"message": f"已获取 {len(raw_materials_pre)} 条素材，达到上限，停止调用后续技能",
"matched_count": len(raw_materials_pre),
})
break
# 技能模式不再走下面的并发/合并逻辑，all_results_nested 留空、跳过 web 通用搜索
all_results_nested: list = []
# 把已收集到的素材挂在 raw_materials_pre，下方合并阶段直接挪过去
else:
search_limit = keyword_count
if extra_keywords:
search_limit = max(search_limit, len(keywords))
search_keywords = keywords[:search_limit]
total = len(search_keywords)
# 每个检索词分到的检索条数 = 总量上限 / 检索词数（向上取整）
per_keyword = _per_keyword_limit(max_materials, total)
_emit(f"检索词已生成，开始检索 {total} 个方向…")
# ── 常规 web/intranet 模式 ──
provider = get_search_provider(ai_writing_config)
# 并发搜索，但每完成一个就推送进度（计数在单线程 asyncio 下无需加锁）
completed = 0
# 按用户指定的关键词数量截断。
# 用户输入 / 追加模式：keywords 本身就是用户填的，全部检索不截断。
if is_user_query_search or is_append_only:
search_keywords = keywords
else:
search_limit = keyword_count
if extra_keywords:
search_limit = max(search_limit, len(keywords))
search_keywords = keywords[:search_limit]
total = len(search_keywords)
per_keyword = _per_keyword_limit(max_materials, total)
_emit(f"检索词已生成，开始检索 {total} 个方向…")
async def _search_one(kw: str) -> list[SearchResult]:
nonlocal completed
try:
return await provider.search(kw, max_results=per_keyword)
finally:
completed += 1
if completed < total:
_emit(f"已完成 {completed}/{total} 次检索，正在继续检索相关内容…")
else:
_emit(f"已完成全部 {total} 次检索")
completed = 0
search_tasks = [_search_one(kw) for kw in search_keywords]
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
async def _search_one(kw: str) -> list[SearchResult]:
nonlocal completed
try:
return await provider.search(kw, max_results=per_keyword)
finally:
completed += 1
if completed < total:
_emit(f"已完成 {completed}/{total} 次检索，正在继续检索相关内容…")
else:
_emit(f"已完成全部 {total} 次检索")
search_tasks = [_search_one(kw) for kw in search_keywords]
all_results_nested = await asyncio.gather(*search_tasks, return_exceptions=True)
# provider 内部吞掉的检索异常（如内网 ES 连接失败），一并取出用于提示用户
search_errors.extend(getattr(provider, "errors", []))
# 合并去重
# - 首次检索 / 普通 re_search（不带 extra_keywords，用户只想 LLM 重出）：旧素材丢弃
# - 素材不足求助 supplement_search：保留旧素材 + 追加新检索结果
# - PAUSE-1 追加检索 is_append_only：保留旧素材 + 仅检索新关键词
seen_urls: set[str] = set()
raw_materials: list[Material] = []
# - 技能检索 is_skill_search：覆盖式替换，旧素材完全丢弃（编排循环已经把素材累积到 raw_materials_pre）
existing_pkg = state.get("material_package")
preserve_existing = existing_pkg and (is_supplement or is_append_only)
if preserve_existing:
for m in existing_pkg.get("materials", []):
seen_urls.add(m.get("url", ""))
raw_materials.append(m)
if is_skill_search:
# 技能编排循环已经完成素材累积，直接用 raw_materials_pre 当 raw_materials
raw_materials: list[Material] = raw_materials_pre # type: ignore[name-defined]
else:
seen_urls: set[str] = set()
raw_materials = []
preserve_existing = existing_pkg and (is_supplement or is_append_only)
if preserve_existing:
for m in existing_pkg.get("materials", []):
seen_urls.add(m.get("url", ""))
raw_materials.append(m)
search_errors: list[str] = []
for results in all_results_nested:
if isinstance(results, Exception):
logger.warning("search failed: %s", results)
search_errors.append(f"{type(results).__name__}: {results}")
continue
for r in results:
if r.url and r.url in seen_urls:
for results in all_results_nested:
if isinstance(results, Exception):
logger.warning("search failed: %s", results)
search_errors.append(f"{type(results).__name__}: {results}")
continue
seen_urls.add(r.url)
raw_materials.append(Material(
id=f"m{len(raw_materials)+1:03d}",
title=r.title,
content=r.content[:500],
source=r.source,
published_at=r.published_at,
relevance_score=r.relevance_score,
source_type=r.source_type,
url=r.url,
recUuid=r.recUuid,
))
# provider 内部吞掉的检索异常（如内网 ES 连接失败），一并取出用于提示用户
search_errors.extend(getattr(provider, "errors", []))
for r in results:
if r.url and r.url in seen_urls:
continue
if r.url:
seen_urls.add(r.url)
raw_materials.append(Material(
id=f"m{len(raw_materials)+1:03d}",
title=r.title,
content=r.content[:500],
source=r.source,
published_at=r.published_at,
relevance_score=r.relevance_score,
source_type=r.source_type,
url=r.url,
recUuid=r.recUuid,
))
# Step 3: 相关度排序（如素材较多，让 LLM 排序）
_emit("检索完成，正在筛选排序素材…")
@@ -264,6 +375,14 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
old_kws = list(existing_pkg.get("keywords") or [])
pkg_keywords = list(dict.fromkeys(old_kws + keywords))
pkg = MaterialPackage(keywords=pkg_keywords, materials=materials, summary=summary)
# 技能检索时把过程步骤一并放进 pkg，前端在素材确认卡里能展示"使用了哪些技能"
if is_skill_search:
pkg["skill_search_used"] = True
pkg["skill_search_steps"] = skill_steps
elif existing_pkg and existing_pkg.get("skill_search_used") and (is_supplement or is_append_only):
# 保留之前一次技能检索的痕迹（不污染纯 web 检索）
pkg["skill_search_used"] = existing_pkg.get("skill_search_used", False)
pkg["skill_search_steps"] = existing_pkg.get("skill_search_steps", [])
# 容错告警：检索接口 / 模型调用出错时以非致命 notice 提示用户，不打断写作流程
notices: list[dict] = []
@@ -274,19 +393,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
if search_errors:
uniq = list(dict.fromkeys(search_errors))
# 解析失败时收集到的接口原始响应体，前端折叠展示便于定位
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
# （技能模式下 provider 局部变量没暴露，所以 raw_dumps 兜底为空列表）
provider_obj = locals().get("provider") or locals().get("web_provider")
raw_dumps = list(dict.fromkeys(getattr(provider_obj, "raw_responses", []) or []))
notices.append(_notice(
"ES 波动导致检索异常",
"；".join(uniq[:3]),
raw="\n\n----------\n\n".join(raw_dumps[:3]),
))
# 进度事件文案：追加模式下让用户看到只搜了哪几个新词
step_start_msg = (
f"追加检索关键词：{', '.join(keywords[:3])}"
if is_append_only
else f"搜索关键词：{', '.join(keywords[:3])}"
)
# 进度事件文案：用户输入 / 追加模式 / 技能模式下让用户看到差异化提示
if is_skill_search:
step_start_msg = f"启用技能检索（{len(skill_steps)} 步）"
elif is_user_query_search:
step_start_msg = f"按你的输入检索：{user_query[:40]}"
elif is_append_only:
step_start_msg = f"追加检索关键词：{', '.join(keywords[:3])}"
else:
step_start_msg = f"搜索关键词：{', '.join(keywords[:3])}"
return {
"material_package": pkg,
# 补充检索：直接回写作（沿用原大纲）；常规检索：进入素材确认暂停点
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/search/__init__.py
+2
View File
@@ -3,6 +3,7 @@ from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
from .skill_search import SkillSearchProvider
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
@@ -60,5 +61,6 @@ __all__ = [
"BaseSearchProvider", "SearchResult",
"InternetSearchProvider", "IntranetSearchProvider",
"DemoSearchProvider", "DuckDuckGoSearchProvider",
"SkillSearchProvider",
"get_search_provider",
]
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/search/skill_search.py
+193
View File
@@ -0,0 +1,193 @@
"""技能选择器：根据用户意图 + 输入，从启用的 Skills 里挑出按相关度排序的若干个。
注意：这个模块**只负责选择技能，不直接产出素材**。
真正的素材产出由 researcher 节点用每个 skill 的描述去编排真实 web/intranet 检索。
流程（对应素材确认卡上的"技能检索"按钮）：
1) list_skills 事件 —— 列出当前启用的全部 skill
2) skill_select_chunk 事件 —— LLM 流式打字 + <think></think> 分离（前端打字机展示）
3) select_skills 事件 —— LLM 给出的排序结果（结构化字段，写到 SkillStepsCard 里）
下游 researcher 节点拿到 selected_skills 后，**按顺序**：
- 用 skill.description + user_query 生成针对性 query
- 调真实 web search provider 执行
- 累积素材达到阈值就停（推 stop_early）
"""
from __future__ import annotations
import logging
from typing import Callable, List, Optional
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..nodes._utils import (
ThinkTagParser,
_content_to_text,
get_model,
parse_json,
safe_get_stream_writer,
)
logger = logging.getLogger(__name__)
_SELECT_SKILLS_SYSTEM = """你是一名情报研究员，需要从可用「技能」中挑出对当前检索任务最有帮助的若干个，并按相关度从高到低排序。
技能是一段封装好的指令 / 知识 / 检索策略，每个技能有 name 和 description。
【输出要求】
- 只输出一个 JSON 对象，不要 markdown 代码块。
- 必须使用英文双引号。
- selected_skills 中的 name 必须严格来自下方"可用技能"列表，不要编造，按相关度从高到低排序。
- 如果用户输入里明确点名了某个技能（按 name 完全匹配），把它排在第一位。
- 至少选 1 个，最多选 {max_select} 个。
【完整示例】
{{"selected_skills": ["platform-sales-data", "industry-report"], "reason": "用户想查各平台销量，平台销量技能最相关"}}"""
_SELECT_SKILLS_USER = """写作意图：{user_intent}
用户补充输入：{user_query}
可用技能（共 {skill_count} 个）：
{skill_list}
请从中按相关度从高到低选出最多 {max_select} 个技能。
只输出 JSON：
{{"selected_skills": ["name1", "name2"], "reason": "为什么选这些 + 排序依据"}}"""
EmitStepFn = Callable[[dict], None]
class SkillPicker:
"""从启用的 Skills 中流式选择按相关度排序的若干个供下游编排使用。
构造参数：
config: LangGraph runnable config（用于创建 model）
model_name: 指定使用的模型，None 则用 configurable.model_name
emit_step: 推送时间线步骤事件（list_skills / select_skills）
max_select: LLM 单次最多挑几个技能
user_query: 用户在素材确认卡输入框写的自然语言
"""
def __init__(
self,
config: RunnableConfig,
*,
model_name: Optional[str] = None,
emit_step: Optional[EmitStepFn] = None,
max_select: int = 5,
user_query: str = "",
) -> None:
self._config = config
self._model_name = model_name
self._emit_step = emit_step or (lambda _step: None)
self._max_select = max_select
self._user_query = (user_query or "").strip()
self._errors: list[str] = []
@property
def errors(self) -> list[str]:
return list(self._errors)
def _load_enabled_skills(self) -> list:
try:
from deerflow.skills import get_or_new_skill_storage
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
except Exception as e:
logger.exception("ai_writing.skill_search: 加载启用技能失败")
self._errors.append(f"skill list error: {e}")
skills = []
self._emit_step({
"type": "list_skills",
"message": f"发现 {len(skills)} 个可用技能",
"skill_names": [s.name for s in skills],
})
return skills
async def pick(self, user_intent: str) -> List[dict]:
"""流式选择并排序候选技能，返回 [{skill, reason}, ...]，按相关度降序。
过程中通过 custom 通道推送 ``skill_select_chunk`` 事件给前端打字机；
结束后再推一条 ``select_skills`` 结构化事件。
"""
skills = self._load_enabled_skills()
if not skills:
self._emit_step({
"type": "select_skills",
"message": "当前没有启用的技能可用",
"skill_names": [],
})
return []
skill_list_text = "\n".join(
f"- {s.name}: {s.description or '(无描述)'}" for s in skills
)
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
# 流式 LLM 调用：与 intent_parser / writer_outline 同套机制
writer = safe_get_stream_writer()
parser = ThinkTagParser()
full_text = ""
def _emit_chunk(text: str, is_thinking: bool) -> None:
writer({
"type": "skill_select_chunk",
"text": text,
"is_thinking": is_thinking,
"agent_name": "素材收集专家",
})
try:
model = get_model(self._config, model_name=self._model_name)
async for chunk in model.astream([
SystemMessage(content=_SELECT_SKILLS_SYSTEM.format(max_select=self._max_select)),
HumanMessage(content=_SELECT_SKILLS_USER.format(
user_intent=user_intent or "(无明确写作意图)",
user_query=self._user_query or "(无)",
skill_count=len(skills),
skill_list=skill_list_text,
max_select=self._max_select,
)),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
_emit_chunk(text, is_thinking)
except Exception as e:
logger.exception("ai_writing.skill_search: 流式选择技能失败")
self._errors.append(f"skill select error: {e}")
result = parse_json(full_text, fallback) if full_text else fallback
wanted_names = result.get("selected_skills") or []
if not isinstance(wanted_names, list):
wanted_names = []
by_name = {s.name: s for s in skills}
reason = result.get("reason", "") or ""
chosen: list[dict] = []
for name in wanted_names:
if not isinstance(name, str):
continue
sk = by_name.get(name.strip())
if sk and sk.name not in {c["skill"].name for c in chosen}:
chosen.append({"skill": sk, "reason": reason})
if len(chosen) >= self._max_select:
break
self._emit_step({
"type": "select_skills",
"message": f"按相关度排序，候选 {len(chosen)} 个技能（将按顺序使用）",
"skill_names": [c["skill"].name for c in chosen],
"reason": reason,
})
return chosen
# 保留旧名字让 search/__init__.py 的导出不报错；研究节点不再用它
SkillSearchProvider = SkillPicker
offline-backend-20260512/backend/packages/harness/deerflow/agents/ai_writing/state.py
+29 -4
View File
@@ -3,21 +3,43 @@ import operator
# ── 素材 ──────────────────────────────────────────────────────────────────────
class Material(TypedDict):
class Material(TypedDict, total=False):
id: str
title: str
content: str # 截断到 500 字
source: str
published_at: str
relevance_score: float
source_type: str # "internet" | "intranet"
source_type: str # "internet" | "intranet" | "skill"
url: str
recUuid: str
# skill 类素材附带的元信息（skill_name / container_path / category），其他类型为空
extra_metadata: dict
class MaterialPackage(TypedDict):
class SkillStep(TypedDict, total=False):
"""技能检索过程的单步记录，用于前端时间线展示。
type=match_skills：根据用户输入流式分析有没有匹配技能（无 skill_names）
type=list_skills：列出可用技能
type=select_skills：LLM 从中选出的相关技能
type=invoke_skill：实际调用某一个技能检索
type=stop_early：达到素材阈值，提前停止后续技能调用
"""
type: str
message: str
skill_names: List[str]
skill_name: str
matched_count: int
reason: str
class MaterialPackage(TypedDict, total=False):
keywords: List[str]
materials: List[Material]
summary: str
# 是否本次素材来自技能检索（覆盖式替换 web 检索）
skill_search_used: bool
# 技能检索的步骤记录（前端展示"使用了 X 技能"卡片）
skill_search_steps: List[SkillStep]
# ── 大纲 ──────────────────────────────────────────────────────────────────────
# 三级结构:章节(##) → 小节(###) → 子要点(bullet -)。
@@ -72,10 +94,13 @@ class ReviewResult(TypedDict):
# ── 用户干预 ──────────────────────────────────────────────────────────────────
class UserInterventionPayload(TypedDict):
class UserInterventionPayload(TypedDict, total=False):
action: str
approved_material_ids: List[str]
extra_keywords: List[str]
# PAUSE-1 素材确认 / enable_skill_search 共用：用户在单输入框里填的自然语言
# （可能是要使用的技能名、想检索的来源、或者一段补充检索意图）。
user_query: str
edited_outline: Optional[Outline]
outline_feedback: str
user_revision_notes: str
offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py
+24 -11
View File
@@ -9,6 +9,7 @@ from deerflow.agents.middlewares.clarification_middleware import ClarificationMi
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
@@ -254,11 +255,14 @@ def _build_middlewares(
List of middleware instances.
"""
resolved_app_config = app_config or get_app_config()
cfg = _get_runtime_config(config)
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
# Inject the admin prompt prefix into the model request (kept out of the
# persisted/displayed message so the user never sees it).
middlewares.append(PromptPrefixMiddleware())
if cfg.get("rag_mode_enabled", False):
middlewares.append(RagCitationMiddleware())
# Add summarization middleware if enabled
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
@@ -266,7 +270,6 @@ def _build_middlewares(
middlewares.append(summarization_middleware)
# Add TodoList middleware if plan mode is enabled
cfg = _get_runtime_config(config)
is_plan_mode = cfg.get("is_plan_mode", False)
is_bootstrap = cfg.get("is_bootstrap", False)
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
@@ -370,6 +373,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
writing_mode = bool(cfg.get("writing_mode", False))
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
@@ -497,24 +501,30 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
is_bootstrap=True,
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
rag_mode_enabled=rag_mode_enabled,
),
state_schema=ThreadState,
)
# Default lead agent (unchanged behavior)
# Use subagent tool groups if in subagent mode, otherwise use parent agent config
effective_tool_groups = subagent_tool_groups if subagent_mode else (agent_config.tool_groups if agent_config else None)
is_notebook_mode = bool(cfg.get("notebook_search_space_id"))
notebook_source_ids: list[int] = cfg.get("notebook_source_ids") or []
default_tools = get_available_tools(
model_name=model_name,
groups=agent_config.tool_groups if agent_config else None,
subagent_enabled=subagent_enabled,
is_scheduled_run=cfg.get("is_scheduled_run", False),
excluded_tools=requested_excluded_tools,
app_config=resolved_app_config,
)
if is_notebook_mode:
from deerflow.tools.builtins.notebook_search_tool import notebook_search_tool
default_tools = [notebook_search_tool] + default_tools
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
tools=get_available_tools(
model_name=model_name,
groups=effective_tool_groups,
subagent_enabled=subagent_enabled,
is_scheduled_run=cfg.get("is_scheduled_run", False),
excluded_tools=requested_excluded_tools,
app_config=resolved_app_config,
),
tools=default_tools,
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
system_prompt=apply_prompt_template(
subagent_enabled=subagent_enabled,
@@ -526,6 +536,9 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
rag_mode_enabled=rag_mode_enabled,
is_notebook_mode=is_notebook_mode,
notebook_source_ids=notebook_source_ids,
),
state_schema=ThreadState,
)
offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py
+120 -8
View File
@@ -667,17 +667,25 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
@lru_cache(maxsize=32)
def _get_cached_skills_prompt_section(
skill_signature: tuple[tuple[str, str, str, str], ...],
display_signature: tuple[tuple[str, str, str], ...],
available_skills_key: tuple[str, ...] | None,
container_base_path: str,
skill_evolution_section: str,
) -> str:
display_by_skill = {name: (mode, custom_instruction) for name, mode, custom_instruction in display_signature}
filtered = [(name, description, category, location) for name, description, category, location in skill_signature if available_skills_key is None or name in available_skills_key]
skills_list = ""
if filtered:
skill_items = "\n".join(
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>\n </skill>"
for name, description, category, location in filtered
)
items = []
for name, description, category, location in filtered:
display_mode, custom_instruction = display_by_skill.get(name, ("", ""))
display_hint = ""
if display_mode:
display_hint = f"\n <display mode=\"{display_mode}\">{custom_instruction}</display>"
items.append(
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>{display_hint}\n </skill>"
)
skill_items = "\n".join(items)
skills_list = f"<available_skills>\n{skill_items}\n</available_skills>"
# Restricted mode: agent has an explicit skill allowlist.
@@ -693,6 +701,21 @@ def _get_cached_skills_prompt_section(
else:
scope_note = ""
display_pattern = (
"""
**Skill Result Display Pattern:**
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
"""
if display_signature
else ""
)
return f"""<skill_system>
You have access to skills that provide optimized workflows for specific tasks. Each skill contains best practices, frameworks, and references to additional resources.
{scope_note}
@@ -702,13 +725,19 @@ You have access to skills that provide optimized workflows for specific tasks. E
3. The skill file contains references to external resources under the same folder
4. Load referenced resources only when needed during execution
5. Follow the skill's instructions precisely
{display_pattern}
{location_hint}{evolution_block}
{skills_list}
</skill_system>"""
def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_config: AppConfig | None = None) -> str:
def get_skills_prompt_section(
available_skills: set[str] | None = None,
*,
app_config: AppConfig | None = None,
display_enabled: bool = False,
) -> str:
"""Generate the skills prompt section with available skills list."""
skills = _get_enabled_skills_for_config(app_config)
@@ -739,11 +768,30 @@ def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_c
return ""
skill_signature = tuple((skill.name, skill.description, skill.category, skill.get_container_file_path(container_base_path)) for skill in skills)
display_signature: tuple[tuple[str, str, str], ...] = ()
if display_enabled:
try:
from deerflow.config.extensions_config import get_extensions_config
extensions = get_extensions_config()
display_signature = tuple(
(
name,
skill_config.display.mode,
skill_config.display.custom_instruction,
)
for name, skill_config in sorted(extensions.skills.items())
if skill_config.display is not None
and skill_config.display.enabled
and skill_config.display.mode != "none"
)
except Exception:
logger.debug("Failed to load skill display configs for prompt injection", exc_info=True)
available_key = tuple(sorted(available_skills)) if available_skills is not None else None
if not skill_signature and available_key is not None:
return ""
skill_evolution_section = _build_skill_evolution_section(effective_skill_evolution_enabled)
return _get_cached_skills_prompt_section(skill_signature, available_key, container_base_path, skill_evolution_section)
return _get_cached_skills_prompt_section(skill_signature, display_signature, available_key, container_base_path, skill_evolution_section)
def get_agent_soul(agent_id: str | None) -> str:
@@ -869,6 +917,9 @@ def apply_prompt_template(
app_config: AppConfig | None = None,
soul_override: str | None = None,
memory_injection_enabled: bool = True,
rag_mode_enabled: bool = False,
is_notebook_mode: bool = False,
notebook_source_ids: list[int] | None = None,
) -> str:
storage_agent_id = agent_id if agent_id is not None else agent_name
@@ -901,7 +952,11 @@ def apply_prompt_template(
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
# Get skills section
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
skills_section = get_skills_prompt_section(
available_skills,
app_config=app_config,
display_enabled=rag_mode_enabled,
)
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
@@ -939,6 +994,21 @@ Writing mode is enabled for this run.
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
network_policy_section = _build_network_policy_section()
rag_mode_section = (
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
When a search, knowledge, skill, or MCP tool returns a result list:
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
</rag_citation_mode>
"""
if rag_mode_enabled
else ""
)
# Format the prompt with dynamic skills and memory
prompt = SYSTEM_PROMPT_TEMPLATE.format(
@@ -956,4 +1026,46 @@ Writing mode is enabled for this run.
acp_section=acp_and_mounts_section,
)
return prompt + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
if is_notebook_mode:
source_ids = notebook_source_ids or []
if source_ids:
source_line = f"当前检索范围：{len(source_ids)} 个来源（IDs: {', '.join(str(i) for i in source_ids)}）"
else:
source_line = "当前检索范围：全部来源"
prompt += f"""
<notebook_mode>
## 角色
你是一个笔记本 AI 助手，**只能**基于笔记本文档来源回答用户问题，禁止使用任何其他知识。
## 可用知识来源
{source_line}
## 强制工作流程（每次用户提问都必须严格执行，无例外）
1. **第一步（必须）**：立即调用 `search_notebook_sources` 工具，`query` 填写用户问题或精炼的检索关键词。
- 无论问题多简单，都必须先检索，不得跳过。
- 若检索结果不足，可再次调用不同 query 补充检索。
2. **第二步**：仅基于检索到的内容组织回答，**不得使用训练知识或推断**。
3. **第三步**：若检索结果中确实无相关内容，明确告知用户"暂未在来源中找到相关信息"。
## 引用格式（强制）
- 每个事实/句子后立即插入 `[[cite:<chunk_id>]]` 标记，chunk_id 来自检索结果中每个文本块的 id 字段。
- 多个来源支持同一句话时连续插入：`[[cite:101]][[cite:102]]`
- 引用标记紧跟被引用内容末尾，不加空格，不单独成行。
## 示例
用户：合同中违约金是多少？
（调用 search_notebook_sources query="违约金"，返回 chunk_id=101 的结果）
回答：根据合同第五条，违约金为合同总额的 0.5%[[cite:101]]，乙方拒绝收货视为违约[[cite:101]]。
## 绝对禁止
- 未调用 `search_notebook_sources` 就直接回答
- 引用不在检索结果中的 chunk_id
- 省略引用标记
- 使用笔记本来源之外的知识
</notebook_mode>"""
return prompt
offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py
+41 -7
View File
@@ -61,6 +61,21 @@ _AUTH_PATTERNS = (
"无权",
"未授权",
)
# Upstream model gateway returned a malformed payload (typically a JSON decode
# error wrapped as `bad_response_status_code`, or a truncated/garbled inference
# response). These are almost always transient inference-side issues and safe
# to retry.
_UPSTREAM_GARBAGE_PATTERNS = (
"bad_response_status_code",
"bad response status",
"expecting ',' delimiter",
"expecting value",
"expecting property name",
"unexpected character",
"unterminated string",
"extra data",
"jsondecodeerror",
)
class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
@@ -139,9 +154,10 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
detail = _extract_error_detail(exc)
lowered = detail.lower()
error_code = _extract_error_code(exc)
error_code_lower = str(error_code).lower() if error_code is not None else ""
status_code = _extract_status_code(exc)
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(str(error_code).lower(), _QUOTA_PATTERNS):
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(error_code_lower, _QUOTA_PATTERNS):
return False, "quota"
if _matches_any(lowered, _AUTH_PATTERNS):
return False, "auth"
@@ -159,6 +175,11 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
return True, "transient"
if _matches_any(lowered, _BUSY_PATTERNS):
return True, "busy"
# Treat upstream gateway garbage (malformed JSON, truncated responses)
# as a retriable transient — the deterministic char-position errors we
# see from vLLM/proxy layers usually clear on a re-roll.
if _matches_any(lowered, _UPSTREAM_GARBAGE_PATTERNS) or _matches_any(error_code_lower, _UPSTREAM_GARBAGE_PATTERNS):
return True, "upstream"
return False, "generic"
@@ -171,21 +192,34 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
seconds = max(1, round(wait_ms / 1000))
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
if reason == "busy":
reason_text = "服务繁忙"
elif reason == "upstream":
reason_text = "上游响应异常"
else:
reason_text = "网络/连接异常"
return f"大模型因{reason_text}调用失败，正在第 {attempt}/{self.retry_max_attempts} 次重试，{seconds} 秒后继续…"
def _build_circuit_breaker_message(self) -> str:
return "大模型因连续多次调用失败已被熔断保护，请稍候再试。"
def _build_user_message(self, exc: BaseException, reason: str) -> str:
detail = _extract_error_detail(exc)
def _build_user_message(self, reason: str) -> str:
if reason == "quota":
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求，请检查模型账户后再试。"
if reason == "auth":
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
if reason in {"busy", "transient"}:
return "大模型因服务繁忙或网络异常，多次重试后仍未恢复,请稍候再继续。"
return f"大模型因「{detail}」原因调用失败。"
if reason == "upstream":
return "大模型当前响应异常（上游模型或网关临时故障），多次重试后仍未恢复，请稍后再试或更换模型。"
# NOTE: We intentionally do NOT embed the raw exception detail into the
# user-facing message. Raw `Error code: 400 - {...}` strings — especially
# Python repr-style dict literals — get persisted as AI messages and then
# re-fed to the model on the next turn, where their unescaped quotes /
# braces can corrupt the upstream JSON payload and cause deterministic
# JSONDecodeError loops. Keep the user message clean; full detail is
# already in the backend log via `logger.warning(..., exc_info=exc)`.
return "大模型本次调用未能成功完成，请稍后再试或更换模型。"
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
try:
@@ -249,7 +283,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(exc, reason))
return AIMessage(content=self._build_user_message(reason))
@override
async def awrap_model_call(
@@ -295,7 +329,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(exc, reason))
return AIMessage(content=self._build_user_message(reason))
def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool:
offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py
+10
View File
@@ -137,7 +137,17 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
背景信息而非用户的新输入。
**跳过开关**:若 ``runtime.context.memory_recall_disabled`` 为真,直接
返回 None。圆桌规划等临时性任务(intent / recommend / leader / special)
不需要从用户个人记忆里召回,Hindsight HTTP 同步阻塞会显著增加 LLM 首
字节延迟(前端"调度中"卡顿);由路由层通过 ``_run_payload`` 的 context
字段统一关闭。
"""
runtime = getattr(request, "runtime", None)
if runtime is not None and getattr(runtime, "context", None):
if runtime.context.get("memory_recall_disabled"):
return None
if not self._injection_enabled:
return None
config = self._memory_config or get_memory_config()
offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/rag_citation_middleware.py
+128
View File
@@ -0,0 +1,128 @@
"""Inject stable citation numbering before RAG final answers."""
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
from collections.abc import Awaitable, Callable
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import SystemMessage
from deerflow.runtime.references import (
build_latest_reference_prompt,
build_reference_debug_payload,
build_reference_debug_summary,
)
logger = logging.getLogger(__name__)
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
def _get_rag_debug_logger() -> logging.Logger:
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
if getattr(debug_logger, "_rag_file_configured", False):
return debug_logger
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
if not log_path.is_absolute():
log_path = Path.cwd() / log_path
log_path.parent.mkdir(parents=True, exist_ok=True)
handler = logging.FileHandler(log_path, encoding="utf-8")
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
debug_logger.addHandler(handler)
debug_logger.setLevel(logging.INFO)
debug_logger.propagate = False
setattr(debug_logger, "_rag_file_configured", True)
setattr(debug_logger, "_rag_file_path", str(log_path))
return debug_logger
def _log_rag_debug(message: str, *args: Any) -> None:
logger.info(message, *args)
_get_rag_debug_logger().info(message, *args)
class RagCitationMiddleware(AgentMiddleware[AgentState]):
"""Give the model the same numbered references the frontend will render."""
@staticmethod
def _enabled(request: ModelRequest) -> bool:
runtime = getattr(request, "runtime", None)
context = getattr(runtime, "context", None) if runtime is not None else None
if isinstance(context, dict) and context.get("rag_mode_enabled"):
return True
config = getattr(runtime, "config", None) if runtime is not None else None
if isinstance(config, dict):
configurable = config.get("configurable")
if isinstance(configurable, dict) and configurable.get("rag_mode_enabled"):
return True
runtime_context = config.get("context")
if isinstance(runtime_context, dict) and runtime_context.get("rag_mode_enabled"):
return True
return False
@staticmethod
def _patch(request: ModelRequest) -> ModelRequest | None:
if not RagCitationMiddleware._enabled(request):
return None
messages = list(request.messages)
prompt, batch = build_latest_reference_prompt(messages)
if not prompt:
_log_rag_debug(
"middleware reference-prompt enabled=true prompt=false messages=%s",
len(messages),
)
return None
summary = build_reference_debug_summary(messages, [batch] if batch else None)
_log_rag_debug(
"middleware reference-prompt enabled=true prompt=true single_batch=true messages=%s "
"tool_messages=%s raw_items=%s normalized_items=%s kept_items=%s latest_batch_id=%s "
"latest_batch_sources=%s",
summary["messages"],
summary["tool_messages"],
summary["raw_items"],
summary["normalized_items"],
summary["kept_items"],
(batch or {}).get("id"),
len((batch or {}).get("sources") or []),
)
_log_rag_debug(
"middleware reference-prompt-payload %s",
json.dumps(
build_reference_debug_payload(
messages=messages,
batch=batch,
prompt=prompt,
),
ensure_ascii=False,
default=str,
),
)
patched_messages = [
*messages,
SystemMessage(content=prompt),
]
return request.override(messages=patched_messages)
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
patched = self._patch(request)
return handler(patched or request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
patched = self._patch(request)
return await handler(patched or request)
offline-backend-20260512/backend/packages/harness/deerflow/client.py
+21 -11
View File
@@ -840,10 +840,13 @@ class DeerFlowClient:
current_config = get_extensions_config()
config_data = {
"mcpServers": mcp_servers,
"skills": {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()},
}
config_data = {
"mcpServers": mcp_servers,
"skills": {
name: skill.model_dump(by_alias=True, exclude_none=True)
for name, skill in current_config.skills.items()
},
}
self._atomic_write_json(config_path, config_data)
@@ -903,13 +906,20 @@ class DeerFlowClient:
if config_path is None:
raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.")
extensions_config = get_extensions_config()
extensions_config.skills[name] = SkillStateConfig(enabled=enabled)
config_data = {
"mcpServers": {n: s.model_dump() for n, s in extensions_config.mcp_servers.items()},
"skills": {n: {"enabled": sc.enabled} for n, sc in extensions_config.skills.items()},
}
extensions_config = get_extensions_config()
current_skill_config = extensions_config.skills.get(name)
extensions_config.skills[name] = SkillStateConfig(
enabled=enabled,
display=current_skill_config.display if current_skill_config else None,
)
config_data = {
"mcpServers": {n: s.model_dump(by_alias=True) for n, s in extensions_config.mcp_servers.items()},
"skills": {
n: sc.model_dump(by_alias=True, exclude_none=True)
for n, sc in extensions_config.skills.items()
},
}
self._atomic_write_json(config_path, config_data)
offline-backend-20260512/backend/packages/harness/deerflow/config/agents_config.py
+52 -2
View File
@@ -1,7 +1,9 @@
"""Configuration and loaders for custom agents."""
import logging
import os
import re
import threading
from typing import Any
import yaml
@@ -13,6 +15,25 @@ logger = logging.getLogger(__name__)
SOUL_FILENAME = "SOUL.md"
AGENT_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
# Per-file mtime caches. These are read on every /api/agents list/detail request
# (one read per agent) — caching keyed by (path, mtime_ns, size) is correct
# because any edit to the file updates mtime, and an external tool re-writing
# the same bytes doesn't actually change the result we'd return. Cache size is
# unbounded but lookup is O(1) and entries are tiny (one parsed dict / string
# per agent file); for the order of N≤10⁴ agents the memory cost is negligible.
_config_cache: dict[str, tuple[tuple[int, int], "AgentConfig | None"]] = {}
_soul_cache: dict[str, tuple[tuple[int, int], str | None]] = {}
_cache_lock = threading.Lock()
def _file_stat_key(path) -> tuple[int, int] | None:
"""Return (mtime_ns, size) for cache invalidation, or None when missing."""
try:
st = os.stat(path)
except OSError:
return None
return (st.st_mtime_ns, st.st_size)
# Backward-compatible alias. Historically this validated the display name;
# after the DB-backed agent table, display names can be Chinese and duplicated.
# The pattern now applies only to storage/runtime ids.
@@ -96,6 +117,18 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
)
return None
# mtime-based cache. The config file is rewritten every time the agent is
# edited via PUT /api/agents/{id}, which bumps mtime — so the next read
# picks up the change automatically. Missing config_file = key None below,
# bypassing the cache (cheap path: just return AgentConfig with id/name).
cache_path_key = str(config_file)
stat_key = _file_stat_key(config_file) if config_file.exists() else None
if stat_key is not None:
with _cache_lock:
cached = _config_cache.get(cache_path_key)
if cached is not None and cached[0] == stat_key:
return cached[1]
data: dict[str, Any] = {}
if config_file.exists():
try:
@@ -114,7 +147,11 @@ def load_agent_config(agent_id: str | None) -> AgentConfig | None:
known_fields = set(AgentConfig.model_fields.keys())
data = {k: v for k, v in data.items() if k in known_fields}
return AgentConfig(**data)
result = AgentConfig(**data)
if stat_key is not None:
with _cache_lock:
_config_cache[cache_path_key] = (stat_key, result)
return result
def load_agent_soul(agent_id: str | None) -> str | None:
@@ -133,8 +170,21 @@ def load_agent_soul(agent_id: str | None) -> str | None:
soul_path = agent_dir / SOUL_FILENAME
if not soul_path.exists():
return None
# mtime-based cache, same rationale as load_agent_config: SOUL.md is
# rewritten on edit (PUT /api/agents/{id}), which bumps mtime.
cache_path_key = str(soul_path)
stat_key = _file_stat_key(soul_path)
if stat_key is not None:
with _cache_lock:
cached = _soul_cache.get(cache_path_key)
if cached is not None and cached[0] == stat_key:
return cached[1]
content = soul_path.read_text(encoding="utf-8").strip()
return content or None
result = content or None
if stat_key is not None:
with _cache_lock:
_soul_cache[cache_path_key] = (stat_key, result)
return result
def list_custom_agents() -> list[AgentConfig]:
offline-backend-20260512/backend/packages/harness/deerflow/config/extensions_config.py
+59 -5
View File
@@ -3,7 +3,7 @@
import json
import os
from pathlib import Path
from typing import Any, Literal
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -53,10 +53,64 @@ class McpServerConfig(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
class SkillDisplayTableColumn(BaseModel):
"""A configured table column for structured skill result display."""
label: str = Field(description="Column label shown in the frontend table")
field: str = Field(description="Field name or dot path read from each result item")
width: str | None = Field(default=None, description="Optional CSS width for this column")
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayConfig(BaseModel):
"""Frontend display adapter for structured skill results."""
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="none",
description="How the frontend should render this skill's structured results",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="doubleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
class ExtensionsConfig(BaseModel):
offline-backend-20260512/backend/packages/harness/deerflow/config/system_settings.py
+44
View File
@@ -17,6 +17,7 @@ import json
import logging
import threading
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -37,10 +38,53 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class AppearanceSettings(BaseModel):
"""System-wide appearance defaults that any *new* user inherits.
Once a user explicitly toggles their theme in the UI, that choice is
persisted in the browser and supersedes this default — so changing
``default_theme`` here only affects users who have never touched the
toggle, and any future first-time loads.
"""
default_theme: Literal["light", "dark"] = Field(
default="light",
description="Theme applied for users who haven't picked their own (light / dark)",
)
model_config = ConfigDict(extra="ignore")
class CitationDisplaySettings(BaseModel):
"""System-wide cleanup rules for citation cards and detail dialogs."""
cleanup_words: list[str] = Field(
default_factory=list,
description="Words or phrases used to remove noisy advertisement/disclaimer lines from citation display.",
)
model_config = ConfigDict(extra="ignore")
class PrivateSquareSettings(BaseModel):
"""Admin-configurable settings for the agent private-square (私密广场).
The password gates read access to all agents flagged ``is_private=True``.
The hash is stored opaquely — the router layer is responsible for hashing
and verifying so the harness package keeps zero dependencies on
``app.gateway.auth``.
"""
enabled: bool = Field(default=False, description="Whether the private square is offered to users at all")
password_hash: str = Field(default="", description="Opaque hash of the admin-configured password. Empty = no password set yet.")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
model_config = ConfigDict(extra="ignore")
offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/__init__.py
+2 -1
View File
@@ -4,10 +4,11 @@ from __future__ import annotations
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.memory import MemoryAgentStore
from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.sql import AgentRepository
__all__ = [
"AgentFavoriteRow",
"AgentRepository",
"AgentRow",
"AgentStore",
offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/base.py
+58
View File
@@ -81,3 +81,61 @@ class AgentStore(ABC):
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
raise NotImplementedError
@abstractmethod
async def list_private(self) -> list[dict[str, Any]]:
"""Return every agent flagged ``is_private=True``.
Caller is responsible for authorization — typically the caller has
already verified the admin-configured private-square password.
"""
raise NotImplementedError
@abstractmethod
async def list_all(self) -> list[dict[str, Any]]:
"""Return every agent regardless of ownership/visibility.
Caller is responsible for authorization — typically the caller has
already verified ``system_role == "admin"``.
"""
raise NotImplementedError
@abstractmethod
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns ``None``
if the agent does not exist.
"""
raise NotImplementedError
@abstractmethod
async def delete_any(self, agent_id: str) -> bool:
"""Delete any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns
``False`` if the agent does not exist.
"""
raise NotImplementedError
# --- Favorites ("收藏") ---
@abstractmethod
async def add_favorite(self, user_id: str, agent_id: str) -> None:
"""Bookmark ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
"""Remove a bookmark. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_favorite_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids ``user_id`` has favorited."""
raise NotImplementedError
@abstractmethod
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/memory.py
+44
View File
@@ -21,6 +21,7 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
@@ -138,3 +139,46 @@ class MemoryAgentStore(AgentStore):
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/model.py
+24 -1
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, String
from sqlalchemy import BigInteger, Boolean, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -21,6 +21,12 @@ class AgentRow(Base):
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# When True, the agent is moved into the password-gated "private square"
# (私密广场) and is hidden from the regular public square / built-in list.
# The flag is independent of ``published``: a private agent is still
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
@@ -29,3 +35,20 @@ class AgentRow(Base):
featured_order: Mapped[int | None] = mapped_column(BigInteger, nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class AgentFavoriteRow(Base):
"""A user's bookmark of a (typically public-square) agent.
Lets a user collect agents from the 广场 (square) into their "我的" tab
without owning them. One row per (user, agent); the unique constraint makes
favoriting idempotent.
"""
__tablename__ = "agent_favorites"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_favorites_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/sql.py
+109 -7
View File
@@ -4,12 +4,14 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import func, or_, select
from sqlalchemy import and_, delete, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
@@ -28,10 +30,31 @@ class AgentRepository(AgentStore):
@staticmethod
def _visible_filter(user_id: str):
"""Visibility for single-record fetches (``get_visible``).
Lenient: a caller who already knows the agent id can fetch it even
when it's flagged ``is_private``. The private-square password is the
*discovery* gate (listings); individual reads are not re-gated, so a
bookmarked / linked private agent stays openable.
"""
return or_(AgentRow.user_id.is_(None), AgentRow.user_id == user_id, AgentRow.published.is_(True))
@staticmethod
def _listing_filter(user_id: str):
"""Visibility for *listings* — hides private-square agents from
non-owners so they only surface inside the password-gated endpoint.
Owners keep seeing their own private agents in '我的'.
"""
return or_(
AgentRow.user_id == user_id,
and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
),
)
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(self._visible_filter(user_id)).order_by(AgentRow.created_at.asc())
stmt = select(AgentRow).where(self._listing_filter(user_id)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
@@ -61,6 +84,7 @@ class AgentRepository(AgentStore):
description=data.get("description") or "",
user_id=data.get("user_id"),
published=bool(data.get("published", False)),
is_private=bool(data.get("is_private", False)),
created_at=now,
updated_at=now,
)
@@ -99,7 +123,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id != user_id:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -112,7 +136,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id is not None:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -157,7 +181,7 @@ class AgentRepository(AgentStore):
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
stmt = (
select(AgentRow)
.where(AgentRow.featured_order.is_not(None), self._visible_filter(user_id))
.where(AgentRow.featured_order.is_not(None), self._listing_filter(user_id))
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
)
async with self._sf() as session:
@@ -168,9 +192,87 @@ class AgentRepository(AgentStore):
if not agent_ids:
return []
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._visible_filter(user_id))
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._listing_filter(user_id))
async with self._sf() as session:
result = await session.execute(stmt)
rows = list(result.scalars())
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
return [_row_to_dict(r) for r in rows]
async def list_private(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(AgentRow.is_private.is_(True)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_all(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete_any(self, agent_id: str) -> bool:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentFavoriteRow.id)
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
# Lost a race against a concurrent favorite of the same agent —
# the unique constraint already guarantees the desired state.
await session.rollback()
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentFavoriteRow).where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_favorite_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260526_01_agents_is_private.py
+62
View File
@@ -0,0 +1,62 @@
"""Add ``agents.is_private`` for the private-square (私密广场) feature.
A nullable-default bool column flagging agents that should live in the
password-gated private square instead of the regular public square.
Revision ID: 20260526_01
Revises: 20260522_01
Create Date: 2026-05-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260526_01"
down_revision = "20260522_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table("agents") and not _has_column("agents", "is_private"):
op.add_column(
"agents",
sa.Column(
"is_private",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
)
if _has_table("agents") and not _has_index("agents", "ix_agents_is_private"):
op.create_index("ix_agents_is_private", "agents", ["is_private"])
def downgrade() -> None:
if _has_index("agents", "ix_agents_is_private"):
op.drop_index("ix_agents_is_private", table_name="agents")
if _has_column("agents", "is_private"):
op.drop_column("agents", "is_private")
offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260528_01_agent_favorites.py
+45
View File
@@ -0,0 +1,45 @@
"""Add ``agent_favorites`` for the agent collection (收藏) feature.
Lets a user bookmark agents from the public square (广场) so they surface in
their "我的" tab without owning them. One row per (user, agent).
Revision ID: 20260528_01
Revises: 20260526_01
Create Date: 2026-05-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260528_01"
down_revision = "20260526_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("agent_favorites"):
op.create_table(
"agent_favorites",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column("agent_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("user_id", "agent_id", name="uq_agent_favorites_user_agent"),
)
op.create_index("ix_agent_favorites_user_id", "agent_favorites", ["user_id"])
op.create_index("ix_agent_favorites_agent_id", "agent_favorites", ["agent_id"])
def downgrade() -> None:
if _has_table("agent_favorites"):
op.drop_table("agent_favorites")
offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260528_02_user_custom_prompts.py
+45
View File
@@ -0,0 +1,45 @@
"""Add ``user_custom_prompts`` for per-user saved prompts (常用提示词).
Bound to the user (not the browser) so prompts follow the account.
Revision ID: 20260528_02
Revises: 20260528_01
Create Date: 2026-05-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260528_02"
down_revision = "20260528_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("user_custom_prompts"):
op.create_table(
"user_custom_prompts",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column("title", sa.String(255), nullable=False, server_default=""),
sa.Column("content", sa.Text(), nullable=False),
sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_user_custom_prompts_user_id", "user_custom_prompts", ["user_id"])
def downgrade() -> None:
if _has_table("user_custom_prompts"):
op.drop_table("user_custom_prompts")
offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
+6 -2
View File
@@ -15,7 +15,7 @@ its storage implementation lives in ``deerflow.runtime.events.store.db`` and
there is no matching entity directory.
"""
from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
@@ -30,15 +30,17 @@ from deerflow.persistence.scheduled_tasks.model import (
ScheduledTaskSubscriptionRow,
SchedulerThreadRow,
)
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
__all__ = [
"FeedbackRow",
"AgentFavoriteRow",
"AgentRow",
"LlmCallMetricRow",
"NotificationRow",
@@ -51,11 +53,13 @@ __all__ = [
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SkillRow",
"SkillDisplayAdapterRow",
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
"UserRow",
]
offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/base.py
+8
View File
@@ -68,6 +68,14 @@ class SkillStore(ABC):
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def list_all(self) -> list[dict[str, Any]]:
"""Return every skill record regardless of ownership/visibility.
Caller is responsible for authorization (admin only).
"""
raise NotImplementedError
@abstractmethod
async def list_published_names(self) -> set[str]:
"""Return the set of skill names with ``published = True`` (any owner).
offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/memory.py
+6
View File
@@ -134,5 +134,11 @@ class MemorySkillStore(SkillStore):
async def count_owned(self, user_id: str) -> int:
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
async def list_all(self) -> list[dict[str, Any]]:
return [
_with_derived(_copy(record)) # type: ignore[arg-type]
for record in self._skills.values()
]
async def list_published_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("published")}
offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/model.py
+13
View File
@@ -31,3 +31,16 @@ class SkillRow(Base):
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class SkillDisplayAdapterRow(Base):
__tablename__ = "skill_display_adapters"
skill_name: Mapped[str] = mapped_column(String(64), primary_key=True)
# Store JSON as text for maximum MySQL compatibility. Some deployments still
# run MySQL variants where native JSON columns or unquoted names such as
# ``display`` are not accepted consistently.
display_config: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/sql.py
+6
View File
@@ -149,6 +149,12 @@ class SkillRepository(SkillStore):
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def list_all(self) -> list[dict[str, Any]]:
stmt = select(SkillRow).order_by(SkillRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_published_names(self) -> set[str]:
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
async with self._sf() as session:
offline-backend-20260512/backend/packages/harness/deerflow/persistence/user_prompts/__init__.py
+18
View File
@@ -0,0 +1,18 @@
"""Per-user custom prompt persistence."""
from __future__ import annotations
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
from deerflow.persistence.user_prompts.sql import UserCustomPromptRepository
__all__ = [
"UserCustomPromptRepository",
"UserCustomPromptRow",
"make_user_prompt_store",
]
def make_user_prompt_store(session_factory):
if session_factory is None:
return None
return UserCustomPromptRepository(session_factory)
offline-backend-20260512/backend/packages/harness/deerflow/persistence/user_prompts/model.py
+23
View File
@@ -0,0 +1,23 @@
"""ORM model for per-user custom prompts (常用提示词)."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class UserCustomPromptRow(Base):
__tablename__ = "user_custom_prompts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
title: Mapped[str] = mapped_column(String(255), nullable=False, default="")
content: Mapped[str] = mapped_column(Text, nullable=False)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
offline-backend-20260512/backend/packages/harness/deerflow/persistence/user_prompts/sql.py
+74
View File
@@ -0,0 +1,74 @@
"""SQLAlchemy-backed per-user custom prompt repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
def _row_to_dict(row: UserCustomPromptRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class UserCustomPromptRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_for_user(self, user_id: str) -> list[dict[str, Any]]:
stmt = (
select(UserCustomPromptRow)
.where(UserCustomPromptRow.user_id == user_id)
.order_by(UserCustomPromptRow.sort_order.asc(), UserCustomPromptRow.created_at.asc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def create(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = UserCustomPromptRow(
id=data["id"],
user_id=user_id,
title=data.get("title") or "",
content=data["content"],
sort_order=data.get("sort_order", 0),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def update(self, prompt_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(UserCustomPromptRow, prompt_id)
if row is None or row.user_id != user_id:
return None
for key in ("title", "content", "sort_order"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete(self, prompt_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(UserCustomPromptRow, prompt_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
offline-backend-20260512/backend/packages/harness/deerflow/runtime/references.py
+865
View File
@@ -0,0 +1,865 @@
"""Build stable per-turn citation batches from LangGraph messages."""
from __future__ import annotations
import json
import re
from typing import Any
from urllib.parse import quote
from deerflow.config.extensions_config import SkillDisplayConfig, get_extensions_config
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
batches: list[dict[str, Any]] = []
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
sources = _collect_sources(messages[index + 1 : end])
if sources:
anchor = _find_reference_batch_anchor(messages, index + 1, end)
message_id = _message_id(anchor) or _message_id(message) or str(index)
batches.append(
{
"id": message_id,
"assistant_message_id": message_id,
"sources": sources,
}
)
return batches
def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
"""Return the single combined citation batch for the latest human turn."""
human_index = _find_latest_human_index(messages)
if human_index is None:
return None
sources = _collect_sources(messages[human_index + 1 :])
if not sources:
return None
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
message_id = _message_id(assistant) or _message_id(messages[human_index]) or str(human_index)
return {
"id": message_id,
"assistant_message_id": message_id,
"source_count": len(sources),
"sources": sources,
}
def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, Any] | None]:
batch = build_latest_reference_batch(messages)
if batch is None:
return "", None
return build_reference_prompt_for_batch(batch), batch
def build_reference_prompt_for_batch(batch: dict[str, Any]) -> str:
"""Build the authoritative model prompt from one already-combined batch."""
sources = [
source
for source in batch.get("sources", [])
if isinstance(source, dict)
]
source_count = len(sources)
lines = [
f"本轮所有检索结果已合并为同一个 referenceBatch，共 {source_count} 条，编号从 [1] 到 [{source_count}] 唯一连续。",
"下面这一批就是当前回答唯一可用的参考文献批次；不要按检索词、工具调用次数或工具原始编号重新分批编号。",
"如果你仍需要补充检索，请只继续调用工具，不要输出正文；最终回答只能引用补充检索后系统重新合并出的最新 referenceBatch。",
"",
f"<reference_batch source_count=\"{source_count}\">",
]
for source in sources:
title = source.get("title") or f"参考文献 {source.get('index')}"
meta = " · ".join(
str(item)
for item in (source.get("source"), source.get("author"), source.get("time"))
if item
)
content = str(source.get("content") or source.get("snippet") or "").strip()
if len(content) > 520:
content = content[:517] + "..."
head = f"[{source.get('index')}] {title}"
if meta:
head += f"（{meta}）"
lines.append(head)
if content:
lines.append(content)
lines.append("")
lines.append("</reference_batch>")
lines.append("")
lines.extend(
[
"引用规则：",
"1. 回答正文中的关键结论必须标注对应编号，例如 [1]、[2]。",
"2. 多个来源可写为 [1][3][5]。",
"3. 只能引用上方存在的编号，不要编造编号。",
"4. 不要使用工具原始编号，也不要重新排序、重新分批或重新从 [1] 开始编号。",
"5. 不要在正文末尾堆完整参考文献列表，前端会展示右侧参考文献。",
"6. 参考文献模式下不要输出 <skill-display .../> 占位符，也不要在正文里额外渲染卡片、表格或列表；前端会在右侧参考文献区域展示。",
]
)
return "\n".join(lines).strip()
def _find_latest_human_index(messages: list[Any]) -> int | None:
for index in range(len(messages) - 1, -1, -1):
if _message_type(messages[index]) == "human":
return index
return None
def build_reference_debug_summary(messages: list[Any], batches: list[dict[str, Any]] | None = None) -> dict[str, Any]:
"""Return compact counts for citation extraction logs."""
turn_summaries: list[dict[str, Any]] = []
total_tool_messages = 0
total_raw_items = 0
total_normalized_items = 0
total_kept_items = 0
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
tool_messages = 0
raw_items = 0
normalized_items = 0
kept_items = 0
for tool_message in messages[index + 1 : end]:
if _message_type(tool_message) != "tool":
continue
tool_messages += 1
items = _first_list(_extract_tool_payload(tool_message))
raw_items += len(items)
for raw_item in items:
normalized = _normalize_reference_item(raw_item, normalized_items + 1)
if not normalized:
continue
normalized_items += 1
kept_items += 1
total_tool_messages += tool_messages
total_raw_items += raw_items
total_normalized_items += normalized_items
total_kept_items += kept_items
turn_summaries.append(
{
"human_index": index,
"tool_messages": tool_messages,
"raw_items": raw_items,
"normalized_items": normalized_items,
"kept_items": kept_items,
}
)
batch_summaries = [
{
"id": batch.get("id"),
"assistant_message_id": batch.get("assistant_message_id"),
"source_count": len(batch.get("sources") or []),
}
for batch in (batches or build_reference_batches(messages))
]
return {
"messages": len(messages),
"turns": len(turn_summaries),
"tool_messages": total_tool_messages,
"raw_items": total_raw_items,
"normalized_items": total_normalized_items,
"kept_items": total_kept_items,
"turns_detail": turn_summaries,
"batches": batch_summaries,
}
def build_reference_debug_payload(
*,
messages: list[Any] | None = None,
batch: dict[str, Any] | None = None,
batches: list[dict[str, Any]] | None = None,
prompt: str | None = None,
) -> dict[str, Any]:
"""Return a JSON-safe payload for comparing model prompts and UI batches."""
payload: dict[str, Any] = {}
if messages is not None:
payload["message_trace"] = [_message_debug_item(index, message) for index, message in enumerate(messages)]
if prompt is not None:
payload["model_reference_prompt"] = prompt
if batch is not None:
payload["batch"] = _batch_debug_item(batch)
if batches is not None:
payload["batches"] = [_batch_debug_item(item) for item in batches]
return payload
def adapt_reference_batches_with_displays(
batches: list[dict[str, Any]],
displays: dict[str, SkillDisplayConfig | dict[str, Any]],
) -> list[dict[str, Any]]:
"""Apply persisted skill display configs to already-built reference batches.
The reference builder can run before the gateway has a fresh extensions
cache, while the skill adapter editor persists the user's latest mapping in
the database. This pass lets the API response use that DB-backed mapping
without changing citation indexes.
"""
normalized_displays: dict[str, SkillDisplayConfig] = {}
for skill_name, display in displays.items():
try:
config = (
display
if isinstance(display, SkillDisplayConfig)
else SkillDisplayConfig.model_validate(display)
)
except Exception:
continue
if not config.enabled or config.mode == "none":
continue
normalized_displays[skill_name] = config
if not normalized_displays:
return batches
displays_by_normalized_name = {
_normalize_name(name): (name, display)
for name, display in normalized_displays.items()
}
adapted_batches: list[dict[str, Any]] = []
for batch in batches:
sources: list[dict[str, Any]] = []
for source in batch.get("sources", []):
if not isinstance(source, dict):
continue
skill_name = str(source.get("skillName") or "")
display = normalized_displays.get(skill_name)
if display is None:
normalized_source = _normalize_name(skill_name)
normalized_match = displays_by_normalized_name.get(normalized_source)
if normalized_match is None:
for normalized_skill, candidate in displays_by_normalized_name.items():
if (
normalized_source.endswith(normalized_skill)
or normalized_skill in normalized_source
):
normalized_match = candidate
break
if normalized_match is not None:
skill_name, display = normalized_match
raw = source.get("raw")
if display is not None and isinstance(raw, dict):
index_value = int(source.get("index") or len(sources) + 1)
configured = _normalize_with_display_config(
raw,
index_value,
skill_name,
display,
)
if configured is not None:
sources.append(
{
**source,
**configured,
"index": index_value,
"skillName": skill_name,
"raw": raw,
}
)
continue
sources.append(source)
adapted_batches.append({**batch, "sources": sources})
return adapted_batches
def _message_debug_item(index: int, message: Any) -> dict[str, Any]:
content = _message_content(message)
tool_calls = _extract_tool_calls(message)
return {
"position": index,
"type": _message_type(message),
"id": _message_id(message),
"name": _message_name(message) if _message_type(message) == "tool" else None,
"tool_call_id": _message_tool_call_id(message) if _message_type(message) == "tool" else None,
"tool_calls": [
{
"id": call.get("id"),
"name": call.get("name"),
"args": call.get("args"),
}
for call in tool_calls
],
"content_length": len(content) if isinstance(content, str) else None,
}
def _batch_debug_item(batch: dict[str, Any]) -> dict[str, Any]:
sources = [source for source in batch.get("sources", []) if isinstance(source, dict)]
return {
"id": batch.get("id"),
"assistant_message_id": batch.get("assistant_message_id"),
"source_count": len(sources),
"sources": [_source_debug_item(source) for source in sources],
}
def _source_debug_item(source: dict[str, Any]) -> dict[str, Any]:
raw = source.get("raw")
raw_record = raw if isinstance(raw, dict) else {}
return {
"index": source.get("index"),
"id": source.get("id"),
"skillName": source.get("skillName"),
"mode": source.get("mode"),
"title": source.get("title"),
"source": source.get("source"),
"author": source.get("author"),
"time": source.get("time"),
"url": source.get("url"),
"snippet": source.get("snippet"),
"content": source.get("content"),
"raw_id_candidates": {
"recUuid": raw_record.get("recUuid"),
"rec_uuid": raw_record.get("rec_uuid"),
"uuid": raw_record.get("uuid"),
"docId": raw_record.get("docId"),
"doc_id": raw_record.get("doc_id"),
"id": raw_record.get("id"),
"url": raw_record.get("url"),
},
"raw": raw,
}
def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
sources: list[dict[str, Any]] = []
pending_tool_contexts: list[dict[str, Any]] = []
tool_contexts_by_id: dict[str, dict[str, Any]] = {}
for message in messages:
message_type = _message_type(message)
if message_type == "ai":
for tool_call in _extract_tool_calls(message):
context = _build_tool_context(tool_call)
pending_tool_contexts.append(context)
if context.get("id"):
tool_contexts_by_id[str(context["id"])] = context
continue
if message_type != "tool":
continue
tool_context = _resolve_tool_context(
message,
pending_tool_contexts,
tool_contexts_by_id,
)
payload = _extract_tool_payload(message)
for raw_item in _first_list(payload):
normalized = _normalize_reference_item(
raw_item,
len(sources) + 1,
payload=payload,
tool_context=tool_context,
)
if not normalized:
continue
normalized["index"] = len(sources) + 1
sources.append(normalized)
return sources
def _find_reference_batch_anchor(messages: list[Any], start: int, end: int) -> Any | None:
answer = _find_last_assistant_answer(messages, start, end)
if answer is not None:
return answer
for index in range(end - 1, start - 1, -1):
candidate = messages[index]
if _message_type(candidate) == "ai" and _message_id(candidate):
return candidate
return None
def _find_last_assistant_answer(messages: list[Any], start: int, end: int) -> Any | None:
for index in range(end - 1, start - 1, -1):
candidate = messages[index]
if (
_message_type(candidate) == "ai"
and _message_id(candidate)
and _has_answer_content(candidate)
):
return candidate
return None
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "")
return str(getattr(message, "type", "") or "")
def _message_id(message: Any) -> str | None:
value = message.get("id") if isinstance(message, dict) else getattr(message, "id", None)
return str(value) if value else None
def _message_content(message: Any) -> Any:
return message.get("content") if isinstance(message, dict) else getattr(message, "content", None)
def _message_name(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
return str(getattr(message, "name", None) or "tool")
def _message_tool_call_id(message: Any) -> str | None:
if isinstance(message, dict):
value = message.get("tool_call_id") or message.get("toolCallId")
else:
value = getattr(message, "tool_call_id", None) or getattr(message, "toolCallId", None)
return str(value) if value else None
def _has_tool_calls(message: Any) -> bool:
if isinstance(message, dict):
direct = message.get("tool_calls")
chunks = message.get("tool_call_chunks")
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
else:
direct = getattr(message, "tool_calls", None)
chunks = getattr(message, "tool_call_chunks", None)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
return bool(direct or chunks or extra)
def _has_answer_content(message: Any) -> bool:
content = _message_content(message)
if isinstance(content, str):
return bool(content.strip())
if isinstance(content, list):
return any(
isinstance(part, dict)
and part.get("type") == "text"
and str(part.get("text") or "").strip()
for part in content
)
return False
def _extract_tool_calls(message: Any) -> list[dict[str, Any]]:
candidates: list[Any] = []
if isinstance(message, dict):
for key in ("tool_calls", "tool_call_chunks"):
value = message.get(key)
if isinstance(value, list):
candidates.extend(value)
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
if isinstance(extra, list):
candidates.extend(extra)
else:
for key in ("tool_calls", "tool_call_chunks"):
value = getattr(message, key, None)
if isinstance(value, list):
candidates.extend(value)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
if isinstance(extra, list):
candidates.extend(extra)
calls: list[dict[str, Any]] = []
seen_ids: set[str] = set()
for raw in candidates:
call = _normalize_tool_call(raw)
if not call:
continue
call_id = str(call.get("id") or "")
if call_id and call_id in seen_ids:
continue
if call_id:
seen_ids.add(call_id)
calls.append(call)
return calls
def _normalize_tool_call(raw: Any) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
fn = raw.get("function") if isinstance(raw.get("function"), dict) else {}
name = raw.get("name") or fn.get("name") or raw.get("tool_name")
if not name:
return None
return {
"id": raw.get("id") or raw.get("tool_call_id") or raw.get("call_id"),
"name": str(name),
"args": _normalize_tool_args(raw.get("args") or raw.get("arguments") or fn.get("arguments")),
}
def _normalize_tool_args(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
if isinstance(parsed, dict):
return parsed
except Exception:
pass
return {"command": value}
return {}
def _build_tool_context(tool_call: dict[str, Any]) -> dict[str, Any]:
tool_name = str(tool_call.get("name") or "tool")
args = tool_call.get("args") if isinstance(tool_call.get("args"), dict) else {}
return {
"id": tool_call.get("id"),
"tool_name": tool_name,
"args": args,
"skill_name": _infer_skill_name(tool_name, args),
}
def _resolve_tool_context(
message: Any,
pending_tool_contexts: list[dict[str, Any]],
tool_contexts_by_id: dict[str, dict[str, Any]],
) -> dict[str, Any]:
tool_call_id = _message_tool_call_id(message)
if tool_call_id and tool_call_id in tool_contexts_by_id:
context = tool_contexts_by_id[tool_call_id]
pending_tool_contexts[:] = [
item for item in pending_tool_contexts if str(item.get("id") or "") != tool_call_id
]
return context
if pending_tool_contexts:
return pending_tool_contexts.pop(0)
tool_name = _message_name(message)
return {
"id": tool_call_id,
"tool_name": tool_name,
"args": {},
"skill_name": _infer_skill_name(tool_name, {}),
}
def _extract_tool_payload(message: Any) -> Any:
content = _message_content(message)
if isinstance(content, dict):
return _unwrap_tool_payload(content)
if isinstance(content, list):
texts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
if isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "json" and isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "text" and isinstance(part.get("text"), str):
texts.append(part["text"])
unwrapped = _unwrap_tool_payload(part)
if unwrapped is not None:
return unwrapped
content = "\n".join(texts)
if isinstance(content, str):
return _parse_json_like(content)
return None
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
list_keys = ("results", "data", "items", "hits", "documents", "records", "sources")
if any(isinstance(value.get(key), list) for key in list_keys):
return value
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
nested = value.get(key)
if isinstance(nested, str):
parsed = _parse_json_like(nested)
if parsed is not None:
return parsed
if isinstance(nested, dict):
unwrapped = _unwrap_tool_payload(nested)
if unwrapped is not None:
return unwrapped
return value
def _parse_json_like(text: str) -> Any:
trimmed = text.strip()
if not trimmed:
return None
try:
return json.loads(trimmed)
except Exception:
pass
first = trimmed.find("{")
last = trimmed.rfind("}")
if 0 <= first < last:
try:
return json.loads(trimmed[first : last + 1])
except Exception:
return None
return None
def _first_list(payload: Any) -> list[Any]:
if isinstance(payload, list):
return payload
if not isinstance(payload, dict):
return []
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
value = payload.get(key)
if isinstance(value, list):
return value
for key in ("result", "payload", "response", "output", "data"):
found = _first_list(payload.get(key))
if found:
return found
return []
def _nested_records(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, dict):
return []
layers = [raw]
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
nested = raw.get(key)
if isinstance(nested, dict):
layers.append(nested)
return layers
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
for layer in layers:
for key in keys:
value = layer.get(key)
if isinstance(value, (str, int, float)) and str(value).strip():
return str(value).strip()
if isinstance(value, list):
text = _stringify(value, "")
if text:
return text
return None
def _enabled_skill_displays() -> dict[str, SkillDisplayConfig]:
try:
return {
name: state.display
for name, state in get_extensions_config().skills.items()
if state.display is not None
and state.enabled
and state.display.enabled
and state.display.mode != "none"
}
except Exception:
return {}
def _infer_skill_name(tool_name: str, args: dict[str, Any]) -> str | None:
displays = _enabled_skill_displays()
if tool_name in displays:
return tool_name
normalized_tool = _normalize_name(tool_name)
for skill_name in displays:
normalized_skill = _normalize_name(skill_name)
if (
normalized_tool == normalized_skill
or normalized_tool.endswith(normalized_skill)
or normalized_skill in normalized_tool
):
return skill_name
for value in args.values():
if not isinstance(value, str):
continue
match = _SKILL_PATH_RE.search(value)
if match:
return match.group(1)
return None
def _resolve_display_match(
raw: dict[str, Any],
payload: Any,
tool_context: dict[str, Any] | None,
) -> tuple[str, SkillDisplayConfig] | None:
displays = _enabled_skill_displays()
if not displays:
return None
skill_name = str((tool_context or {}).get("skill_name") or "")
display = displays.get(skill_name)
if display is not None:
return skill_name, display
tool_name = str((tool_context or {}).get("tool_name") or "")
inferred = _infer_skill_name(tool_name, (tool_context or {}).get("args") or {})
if inferred and inferred in displays:
return inferred, displays[inferred]
return None
def _normalize_with_display_config(
raw: dict[str, Any],
fallback_index: int,
skill_name: str,
display: SkillDisplayConfig,
) -> dict[str, Any] | None:
title = _stringify(_get_by_path(raw, display.title_field), f"结果 {fallback_index}")
snippet = _stringify(_get_by_path(raw, display.summary_field), "")
content = _stringify(_get_by_path(raw, display.content_field), snippet)
source = _stringify(_get_by_path(raw, display.source_field), "") or None
author = _stringify(_get_by_path(raw, display.author_field), "") or None
item_type = _stringify(_get_by_path(raw, display.type_field), "result") or "result"
time = _stringify(_get_by_path(raw, display.time_field), "") or None
item_id = _stringify(_get_by_path(raw, display.id_field), "") or None
score = _number_or_none(_get_by_path(raw, display.score_field))
url = _resolve_display_link(raw, display)
if not title and not content and not url:
return None
return {
"id": item_id or url,
"skillName": skill_name,
"mode": display.mode,
"title": title,
"type": item_type,
"snippet": snippet,
"content": content,
"source": source,
"author": author,
"time": time,
"url": url,
"score": score,
"tableColumns": [column.model_dump(by_alias=True) for column in display.table_columns],
"raw": raw,
}
def _resolve_display_link(raw: dict[str, Any], display: SkillDisplayConfig) -> str | None:
url = _stringify(_get_by_path(raw, display.url_field), "")
item_id = _stringify(_get_by_path(raw, display.id_field), "")
if display.link_strategy == "url":
return url or None
if display.link_strategy == "id":
return _render_template(display.link_template, raw) if item_id and display.link_template else None
return url or (_render_template(display.link_template, raw) if item_id and display.link_template else None) or None
def _render_template(template: str | None, raw: dict[str, Any]) -> str | None:
if not template:
return None
def replace(match: re.Match[str]) -> str:
value = _get_by_path(raw, match.group(1).strip())
return quote(_stringify(value, ""), safe="")
return re.sub(r"\{([^}]+)\}", replace, template)
def _get_by_path(value: Any, path: str | None) -> Any:
if not path:
return None
cursor = value
for part in path.split("."):
if isinstance(cursor, dict):
cursor = cursor.get(part)
else:
return None
return cursor
def _stringify(value: Any, fallback: str = "") -> str:
if value is None:
return fallback
if isinstance(value, (str, int, float)):
text = str(value).strip()
return text if text else fallback
if isinstance(value, list):
parts = [
str(item).strip()
for item in value
if isinstance(item, (str, int, float)) and str(item).strip()
]
return "、".join(parts) if parts else fallback
return fallback
def _number_or_none(value: Any) -> float | None:
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
try:
return float(value)
except Exception:
return None
return None
def _normalize_name(name: str) -> str:
return re.sub(r"[^a-z0-9]+", "", name.lower())
def _normalize_reference_item(
raw: Any,
fallback_index: int,
*,
payload: Any = None,
tool_context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
display_match = _resolve_display_match(raw, payload, tool_context)
if display_match is not None:
skill_name, display = display_match
configured = _normalize_with_display_config(raw, fallback_index, skill_name, display)
if configured is not None:
return configured
layers = _nested_records(raw)
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
author = _pick(layers, ("author", "authors", "creator", "writer", "byline", "publisher", "owner"))
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
score_raw = _pick(layers, ("score", "relevance", "similarity"))
score = None
if score_raw is not None:
try:
score = float(score_raw)
except Exception:
score = None
if not title:
title = source or f"结果 {fallback_index}"
if not content:
content = ""
if not title and not content and not url:
return None
return {
"id": item_id or url,
"skillName": str((tool_context or {}).get("skill_name") or (tool_context or {}).get("tool_name") or "backend-reference"),
"mode": "citation",
"title": title,
"type": "result",
"snippet": content,
"content": content,
"source": source,
"author": author,
"time": time,
"url": url,
"score": score,
"raw": raw,
}
def _reference_key(source: dict[str, Any]) -> str:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
item_id = source.get("id") or source.get("url") or raw.get("recUuid") or raw.get("uuid") or raw.get("docId") or raw.get("id")
if item_id:
return str(item_id).casefold().strip()
return "|".join(
str(source.get(key) or "").casefold().strip()
for key in ("title", "source", "time", "content", "snippet")
)
offline-backend-20260512/backend/packages/harness/deerflow/runtime/scheduler/__init__.py
+8 -2
View File
@@ -1,4 +1,10 @@
from .registry import get_scheduled_task_service, set_scheduled_task_service
from .service import DuplicateTaskNameError, ScheduledTaskService
from .service import DuplicateTaskNameError, MarkdownRequirementError, ScheduledTaskService
__all__ = ["DuplicateTaskNameError", "ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
__all__ = [
"DuplicateTaskNameError",
"MarkdownRequirementError",
"ScheduledTaskService",
"get_scheduled_task_service",
"set_scheduled_task_service",
]
offline-backend-20260512/backend/packages/harness/deerflow/runtime/scheduler/service.py
+77
View File
@@ -36,6 +36,27 @@ _RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
# Minimum length (in characters) the generated Markdown file's content must
# reach when a task has ``require_markdown`` enabled. The model is NOT told about
# this threshold — it is enforced silently by the service.
_MARKDOWN_MIN_CHARS = 100
# Instruction appended to the prompt when ``require_markdown`` is on. It only
# asks the model to produce a ``.md`` file; the length requirement is enforced
# in code (not surfaced to the model).
_MARKDOWN_INSTRUCTION = (
"\n\n【输出要求】请将本次任务的最终结果生成为一个 Markdown(.md) 格式的文件，"
"保存到 /mnt/user-data/outputs 目录，并使用 present_files 展示该文件。"
)
class MarkdownRequirementError(Exception):
"""Raised when a ``require_markdown`` task did not produce a valid .md file.
The run is marked failed (no retry) so the user can see the output did not
meet the requirement (a Markdown file longer than the minimum length).
"""
class DuplicateTaskNameError(Exception):
"""A user already owns a scheduled task with the requested name.
@@ -166,6 +187,42 @@ def _looks_like_markdown(content: str) -> bool:
return any(re.search(pattern, content, flags=re.MULTILINE) for pattern in markdown_patterns)
def _task_requires_markdown(task: dict[str, Any]) -> bool:
"""Whether a task has the ``require_markdown`` flag set.
The flag lives inside ``execution_context`` (no dedicated column) so the
feature works without a schema migration on existing databases.
"""
ctx = task.get("execution_context")
return bool(isinstance(ctx, dict) and ctx.get("require_markdown"))
def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
"""True when the run produced a ``.md`` file whose text exceeds the minimum.
For tasks with ``require_markdown`` enabled the run only succeeds if at least
one generated file is Markdown (``.md``/``.markdown``) and its content is
longer than :data:`_MARKDOWN_MIN_CHARS` characters. The file is read from its
archived path; binary/undecodable files never qualify.
"""
for item in result_files or []:
if not isinstance(item, dict):
continue
name = str(item.get("name") or "")
if not name.lower().endswith((".md", ".markdown")):
continue
path = item.get("path")
if not isinstance(path, str) or not path:
continue
try:
text = Path(path).read_text(encoding="utf-8", errors="ignore")
except Exception:
continue
if len(text.strip()) > _MARKDOWN_MIN_CHARS:
return True
return False
def _build_result(
task: dict[str, Any],
*,
@@ -451,6 +508,7 @@ class ScheduledTaskService:
source_agent_name: str | None = None,
execution_agent_name: str | None = None,
execution_context: dict[str, Any] | None = None,
require_markdown: bool = False,
) -> dict[str, Any]:
tz = _normalize_timezone(timezone)
cron_expr = _schedule_to_cron(schedule)
@@ -461,6 +519,7 @@ class ScheduledTaskService:
context = dict(execution_context or {})
if execution_agent_name:
context["agent_name"] = execution_agent_name
context["require_markdown"] = bool(require_markdown)
normalized_description = description.strip() if isinstance(description, str) else None
task = await self.store.create_task(
{
@@ -519,6 +578,18 @@ class ScheduledTaskService:
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
if data.get("execution_context") is not None:
update_data["execution_context"] = data["execution_context"]
# ``require_markdown`` is persisted inside execution_context (no dedicated
# column), so merge it into the context being written — preferring a
# context supplied in the same request, else the task's existing one.
if data.get("require_markdown") is not None:
base_ctx = (
update_data["execution_context"]
if "execution_context" in update_data
else (existing or {}).get("execution_context")
)
ctx = dict(base_ctx or {})
ctx["require_markdown"] = bool(data["require_markdown"])
update_data["execution_context"] = ctx
# Re-enabling a task must also schedule it forward from now, otherwise a
# paused task with a stale/empty next_run_at would never fire again.
if update_data.get("enabled") is True and "next_run_at" not in update_data:
@@ -831,6 +902,10 @@ class ScheduledTaskService:
run_id=run_id,
output_snapshot=output_snapshot,
)
if _task_requires_markdown(task) and not _has_valid_markdown_file(result_files):
raise MarkdownRequirementError(
f"未满足输出要求：需生成 Markdown(.md) 文件，且文件正文不少于 {_MARKDOWN_MIN_CHARS} 字"
)
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
@@ -895,6 +970,8 @@ class ScheduledTaskService:
f"计划时间：{scheduled_for.astimezone(ZoneInfo(task.get('timezone') or 'UTC')).isoformat()}\n\n"
f"{task['prompt']}"
)
if execution_context.get("require_markdown"):
content += _MARKDOWN_INSTRUCTION
body = RunCreateRequest(
assistant_id="lead_agent",
input={"messages": [{"type": "human", "content": content}]},
offline-backend-20260512/backend/packages/harness/deerflow/sandbox/local/local_sandbox.py
+48 -27
View File
@@ -37,14 +37,26 @@ class LocalSandbox(Sandbox):
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
@@ -302,30 +314,39 @@ class LocalSandbox(Sandbox):
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [shell, "-NoProfile", "-Command", resolved_command]
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
timeout=600,
)
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
output = result.stdout
if result.stderr:
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
offline-backend-20260512/backend/packages/harness/deerflow/skills/cascade.py
+55
View File
@@ -149,3 +149,58 @@ async def cascade_skill_unpublished_or_deleted(
counters["notified"] += 1
return counters
def sync_agent_skills_to_available(available_skill_names: set[str]) -> dict[str, Any]:
"""Strip skill references that no longer exist on disk from every agent.
Walks every agent dir and rewrites its ``config.yaml`` ``skills`` list to
contain only names that appear in ``available_skill_names``. Used by the
admin "agent-skills sync" endpoint to repair historical drift — skills
deleted before the cascade existed, deleted via a path that skipped the
cascade, or removed from the filesystem manually.
Synchronous because it only touches local files and the admin endpoint
runs it in a worker thread.
Returns ``{checked, modified, removals: [{agent_id, removed_skills}]}``.
"""
counters: dict[str, Any] = {"checked": 0, "modified": 0, "removals": []}
for agent_dir in _list_agent_dirs():
config_file = agent_dir / "config.yaml"
if not config_file.exists():
continue
counters["checked"] += 1
try:
with config_file.open("r", encoding="utf-8") as f:
data: dict[str, Any] = yaml.safe_load(f) or {}
except yaml.YAMLError:
logger.warning("Sync: failed to parse %s, skipping", config_file)
continue
skills = data.get("skills")
if not isinstance(skills, list) or not skills:
continue
kept: list[str] = []
removed: list[str] = []
for name in skills:
if isinstance(name, str) and name in available_skill_names:
kept.append(name)
else:
removed.append(name if isinstance(name, str) else str(name))
if not removed:
continue
data["skills"] = kept
with config_file.open("w", encoding="utf-8") as f:
yaml.safe_dump(data, f, default_flow_style=False, allow_unicode=True)
counters["modified"] += 1
counters["removals"].append({"agent_id": agent_dir.name, "removed_skills": removed})
logger.info("Sync: removed orphan skills %s from agent '%s'", removed, agent_dir.name)
return counters
offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/notebook_search_tool.py
+71
View File
@@ -0,0 +1,71 @@
"""笔记本 RAG 检索工具。
从 LangGraph configurable 中读取 notebook_search_space_id / notebook_source_ids /
notebook_token，调用外部检索接口返回相关文本块，供 LLM 在笔记本模式下引用回答。
API base URL 通过环境变量 DEER_FLOW_NOTEBOOK_API_BASE 配置。
"""
from __future__ import annotations
import logging
import os
import httpx
from langchain.tools import tool
from langgraph.config import get_config
logger = logging.getLogger(__name__)
_SEARCH_PATH = "/api/v1/source/mentioned-documents/search"
def _notebook_api_base() -> str:
return (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
@tool("search_notebook_sources", parse_docstring=True)
def notebook_search_tool(query: str, top_k: int = 10) -> str:
"""Search the notebook's knowledge sources to find relevant content.
Call this tool for EVERY user question. After retrieving results, cite
sources inline using [[cite:<document_id>]] markers immediately after
each claim, e.g. "违约金为0.5%[[cite:101]]".
Args:
query: The user's question or a refined search query.
top_k: Number of top chunks to retrieve (default 10).
"""
base = _notebook_api_base()
if not base:
return "Notebook API base URL not configured (DEER_FLOW_NOTEBOOK_API_BASE)."
cfg = get_config().get("configurable", {})
search_space_id: int | None = cfg.get("notebook_search_space_id")
source_ids: list[int] = cfg.get("notebook_source_ids") or []
token: str = cfg.get("notebook_token") or ""
if not search_space_id:
return "No notebook search space configured."
logger.info("notebook_search_tool query=%r source_ids=%s search_space_id=%s", query, source_ids, search_space_id)
try:
resp = httpx.post(
f"{base}{_SEARCH_PATH}",
headers={"Authorization": f"Bearer {token}"},
json={
"query": query,
"source_ids": [source_ids],
"search_space_id": search_space_id,
"top_k": top_k,
},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
content = data.get("content") if isinstance(data, dict) else None
return content if content else "No relevant content found."
except Exception as e:
logger.warning("notebook_search_tool failed: %s", e)
return f"Search failed: {e}"
offline-backend-20260512/backend/packages/harness/tests/test_rag_references.py
+96
View File
@@ -0,0 +1,96 @@
"""Tests for RAG citation batch construction."""
from __future__ import annotations
import json
from deerflow.runtime.references import (
build_latest_reference_batch,
build_latest_reference_prompt,
build_reference_batches,
)
def _tool_message(call_id: str, query: str, start: int) -> dict:
return {
"type": "tool",
"name": "web_search",
"tool_call_id": call_id,
"content": json.dumps(
{
"query": query,
"results": [
{
"title": f"{query} result {start}",
"content": f"{query} content {start}",
"url": f"https://example.test/{start}",
},
{
"title": f"{query} result {start + 1}",
"content": f"{query} content {start + 1}",
"url": f"https://example.test/{start + 1}",
},
],
},
ensure_ascii=False,
),
}
def test_latest_reference_prompt_uses_one_combined_batch_for_multi_search_turn() -> None:
messages = [
{"type": "human", "id": "human-1", "content": "search two terms"},
{
"type": "ai",
"id": "ai-search-1",
"content": "",
"tool_calls": [{"id": "call-1", "name": "web_search", "args": {"query": "Trump"}}],
},
_tool_message("call-1", "Trump", 1),
{
"type": "ai",
"id": "ai-search-2",
"content": "",
"tool_calls": [{"id": "call-2", "name": "web_search", "args": {"query": "White House"}}],
},
_tool_message("call-2", "White House", 3),
]
batch = build_latest_reference_batch(messages)
assert batch is not None
assert batch["source_count"] == 4
assert [source["index"] for source in batch["sources"]] == [1, 2, 3, 4]
assert [source["title"] for source in batch["sources"]] == [
"Trump result 1",
"Trump result 2",
"White House result 3",
"White House result 4",
]
prompt, prompt_batch = build_latest_reference_prompt(messages)
assert prompt_batch == batch
assert '<reference_batch source_count="4">' in prompt
assert "[1] Trump result 1" in prompt
assert "[4] White House result 4" in prompt
assert "重新分批" in prompt
def test_reference_batches_keep_separate_user_turns_but_merge_searches_inside_turn() -> None:
messages = [
{"type": "human", "id": "human-1", "content": "first"},
{"type": "ai", "id": "ai-1", "content": "", "tool_calls": [{"id": "c1", "name": "web_search"}]},
_tool_message("c1", "first", 1),
{"type": "ai", "id": "answer-1", "content": "answer [1]"},
{"type": "human", "id": "human-2", "content": "second"},
{"type": "ai", "id": "ai-2", "content": "", "tool_calls": [{"id": "c2", "name": "web_search"}]},
_tool_message("c2", "second-a", 3),
{"type": "ai", "id": "ai-3", "content": "", "tool_calls": [{"id": "c3", "name": "web_search"}]},
_tool_message("c3", "second-b", 5),
{"type": "ai", "id": "answer-2", "content": "answer [1][4]"},
]
batches = build_reference_batches(messages)
assert [len(batch["sources"]) for batch in batches] == [2, 4]
assert [source["index"] for source in batches[1]["sources"]] == [1, 2, 3, 4]
assert batches[0]["assistant_message_id"] == "answer-1"
assert batches[1]["assistant_message_id"] == "answer-2"
offline-backend-20260512/backend/tests/test_agents_backend.py
+8 -6
View File
@@ -92,7 +92,7 @@ async def test_update_builtin_only_targets_builtins() -> None:
@pytest.mark.asyncio
async def test_get_editable_or_raise_admin_can_edit_builtin() -> None:
"""Route helper: admin gets edit access to built-ins; non-admin still 403."""
"""Route helper: admin gets edit access to any agent; non-admin still 403."""
from fastapi import HTTPException
from app.gateway.routers.agents import _get_editable_or_raise
@@ -118,14 +118,16 @@ async def test_get_editable_or_raise_admin_can_edit_builtin() -> None:
await _get_editable_or_raise(store, "builtin-agent", "user-a", is_admin=False)
assert exc.value.status_code == 403
# Admin tries to edit someone else's PUBLISHED custom agent → still 403
# (admin override only applies to built-ins, not to others' custom agents).
# Admin edits someone else's custom agent → returns ('admin', record).
# Behavior changed: admins manage the full agent catalog (built-in + any
# user's agents), not just built-ins. Non-admins are still owner-only.
await store.create(
{"id": "published-other", "name": "Pub", "user_id": "user-x", "published": True}
)
with pytest.raises(HTTPException) as exc:
await _get_editable_or_raise(store, "published-other", "user-a", is_admin=True)
assert exc.value.status_code == 403
record, mode = await _get_editable_or_raise(
store, "published-other", "user-a", is_admin=True
)
assert mode == "admin" and record["id"] == "published-other"
def test_build_run_config_injects_custom_assistant_as_agent_id() -> None: