Merge remote-tracking branch 'origin/master' into ji

This commit is contained in:
吉鑫升 committed 2026-06-11 16:39:30 +08:00
commit 5daa4b3e20
62 files changed
+4230 -1025

No files matched your search

@@ -109,20 +109,42 @@ async def _search_notebook(
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
import httpx # 懒导入:仅在笔记模式下才需要
# 入参全 dump(token 只打长度,不打明文,防止泄漏到日志)
logger.info(
"[notebook-search] called keywords=%s search_space_id=%s source_ids=%s token_len=%d",
keywords,
search_space_id,
source_ids,
len(token) if token else 0,
)
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
if not base:
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
# 改成 error 级别,warning 在很多部署里默认会被过滤掉看不到
logger.error(
"[notebook-search] DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search. "
"请在后端启动环境里设这个环境变量(指向笔记后端根地址,例如 http://localhost:8003)。"
)
return []
full_url = f"{base}{_NOTEBOOK_SEARCH_PATH}"
logger.info("[notebook-search] base=%s full_url=%s", base, full_url)
results: list[Material] = []
mat_idx = 0
try:
async with httpx.AsyncClient(timeout=20) as client:
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
"query": kw,
"source_ids": source_ids,
"search_space_id": search_space_id,
"top_k": _NOTEBOOK_TOP_K,
})
try:
# /mentioned-documents/search 接口无需鉴权,不传 Authorization header
resp = await client.post(
f"{base}{_NOTEBOOK_SEARCH_PATH}",
headers={"Authorization": f"Bearer {token}"},
full_url,
json={
"query": kw,
"source_ids": source_ids,
@@ -130,10 +152,29 @@ async def _search_notebook(
"top_k": _NOTEBOOK_TOP_K,
},
)
logger.info(
"[notebook-search] kw=%r http_status=%d body_len=%d",
kw,
resp.status_code,
len(resp.content or b""),
)
if resp.status_code >= 400:
# 把错误响应体打出来,方便定位 401 / 422 等
logger.error(
"[notebook-search] kw=%r HTTP %d response: %s",
kw,
resp.status_code,
(resp.text or "")[:1000],
)
resp.raise_for_status()
data = resp.json()
# 解析 documents 列表,每个 chunk 作为独立素材
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
logger.info(
"[notebook-search] kw=%r parsed documents=%d",
kw,
len(documents or []),
)
for doc in documents or []:
doc_title: str = doc.get("title") or "笔记文档"
doc_url: str = doc.get("url") or ""
@@ -154,10 +195,11 @@ async def _search_notebook(
url=doc_url,
))
except Exception as e:
logger.warning("notebook search kw=%r failed: %s", kw, e)
logger.warning("[notebook-search] kw=%r failed: %s", kw, e, exc_info=True)
except Exception as e:
logger.warning("notebook search client error: %s", e)
logger.warning("[notebook-search] client error: %s", e, exc_info=True)
logger.info("[notebook-search] done — returning %d materials", len(results))
return results
@@ -250,6 +292,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
notebook_materials: list[Material] = []
if notebook_search_space_id:
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
notebook_token: str = cfg.get("notebook_token") or ""
@@ -369,6 +412,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
all_results_nested: list = []
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
elif material_source == "notebook" and notebook_search_space_id:
# ── 纯笔记模式 ──
# 用户在表单选了「我的空间」作为素材来源 → 跳过 web/intranet 检索,
# 仅使用 Step 1.5 已拉取的 notebook_materials。
# 与 UI 文案"从空间中检索素材"对齐。
_emit(f"仅使用笔记素材,跳过 web/intranet 检索({len(notebook_materials)} 条)")
all_results_nested = []
else:
# ── 常规 web/intranet 模式 ──
provider = get_search_provider(ai_writing_config)
@@ -331,6 +331,13 @@ def _build_middlewares(
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents))
# SkillReviewMiddleware — trigger background skill review after each turn.
# Gated by skill_evolution.enabled + skill_evolution.review_worker.enabled.
if resolved_app_config.skill_evolution.enabled and resolved_app_config.skill_evolution.review_worker.enabled:
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
middlewares.append(SkillReviewMiddleware())
# LoopDetectionMiddleware — detect and break repetitive tool call loops
middlewares.append(LoopDetectionMiddleware())
@@ -1194,7 +1194,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
<notebook_mode>
## 角色
你是一个笔记本 AI 助手,**只能**基于笔记本文档来源回答用户问题,禁止使用任何其他知识。
你是一个空间 AI 助手,**只能**基于空间文档来源回答用户问题,禁止使用任何其他知识。
## 可用知识来源
{source_line}
@@ -1220,7 +1220,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
- 未调用 `search_notebook_sources` 就直接回答
- 引用不在检索结果中的 chunk_id
- 省略引用标记
- 使用笔记本来源之外的知识
- 使用空间来源之外的知识
</notebook_mode>"""
return prompt
@@ -0,0 +1,54 @@
"""SkillReviewMiddleware — triggers background skill review after each agent turn.
Hooks into ``after_agent``: once the lead agent graph has finished a full turn,
this middleware counts the tool calls made and, if the threshold is met, hands
the conversation messages to the SkillReviewWorker as a background asyncio task.
No blocking I/O or LLM calls happen in-band — the review runs fully in the
background and never delays the HTTP response.
"""
from __future__ import annotations
import logging
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langgraph.runtime import Runtime
from deerflow.config.app_config import get_app_config
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.skills.review_worker import maybe_start_review
logger = logging.getLogger(__name__)
class SkillReviewMiddlewareState(AgentState):
"""Minimal state schema — only needs ``messages``."""
pass
class SkillReviewMiddleware(AgentMiddleware[SkillReviewMiddlewareState]):
"""Dispatch a background skill review after every substantial agent turn.
'Substantial' is defined by ``skill_evolution.review_worker.min_tool_calls``
(default 5). Turns below the threshold are silently skipped to avoid
wasting LLM tokens on trivial exchanges.
"""
state_schema = SkillReviewMiddlewareState
@override
def after_agent(self, state: Any, runtime: Runtime) -> None:
"""Fire background review; never raises — failures are logged only."""
try:
app_config = get_app_config()
messages = state.get("messages", []) if hasattr(state, "get") else []
thread_id: str = runtime.context.get("thread_id", "") if runtime.context else ""
if not thread_id:
return
user_id = get_effective_user_id()
maybe_start_review(thread_id, messages, app_config, user_id)
except Exception:
logger.exception("SkillReviewMiddleware.after_agent: unexpected error (ignored)")
@@ -107,7 +107,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
content=last_message.content,
id=last_message.id,
name=last_message.name,
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
additional_kwargs={**last_message.additional_kwargs, "run_id": (runtime.context or {}).get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
)
return {
@@ -1,30 +1,39 @@
from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
enabled: bool = Field(
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
from pydantic import BaseModel, Field
class ReviewWorkerConfig(BaseModel):
"""Configuration for the post-turn background skill review worker."""
enabled: bool = Field(default=False, description="Whether the review worker is active.")
min_tool_calls: int = Field(default=5, description="Minimum tool calls in a turn to trigger review. Matches the prompt's '≥5 steps' hard condition so simple queries never reach the LLM reviewer.")
max_messages: int = Field(default=60, description="Max messages passed to the review LLM.")
model_name: str | None = Field(default=None, description="Model to use for review; defaults to primary model.")
timeout_seconds: int = Field(default=120, description="Overall timeout for a single review run.")
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
enabled: bool = Field(
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
review_worker: ReviewWorkerConfig = Field(
default_factory=ReviewWorkerConfig,
description="Configuration for the post-turn background skill review worker.",
)
@@ -135,8 +135,18 @@ def resolve_user_path(user_id: str) -> Path:
return get_paths().user_dir(user_id) / _FILENAME
# Only these curator sub-fields may be overridden per user.
# All other fields (enabled, model_name, timeouts, etc.) are global-only.
_USER_ALLOWED_CURATOR_KEYS = {"interval_hours", "stale_after_days", "archive_after_days", "model_name"}
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid).
Only the three curator threshold keys are honoured; any other keys
(e.g. legacy ``enabled`` or ``model_name``) are silently stripped so that
old files written by previous versions cannot override global-only settings.
"""
path = resolve_user_path(user_id)
if not path.exists():
return None
@@ -154,7 +164,14 @@ def load_user_overrides(user_id: str) -> dict[str, Any] | None:
if not isinstance(data, dict):
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
return None
return data
# Strip top-level keys that are not user-configurable (e.g. legacy "enabled").
curator_raw = data.get("curator")
if not isinstance(curator_raw, dict):
return None
filtered_curator = {k: v for k, v in curator_raw.items() if k in _USER_ALLOWED_CURATOR_KEYS}
if not filtered_curator:
return None
return {"curator": filtered_curator}
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
@@ -0,0 +1,376 @@
"""Post-turn background skill review worker.
After a conversation turn completes, SkillReviewMiddleware calls
``maybe_start_review()``. This module runs a lightweight LLM React agent
(with skill_manage + skill_list tools) that reads the conversation history
and decides whether to create or patch skills.
Concurrency model: per-thread asyncio.Lock + a single _pending slot.
If a review is already running for the thread, the newer messages overwrite
the pending slot; once the current review finishes, it drains that slot.
"""
from __future__ import annotations
import asyncio
import logging
import textwrap
from typing import TYPE_CHECKING, Any
from langchain_core.messages import BaseMessage
if TYPE_CHECKING:
from deerflow.config.app_config import AppConfig
logger = logging.getLogger(__name__)
_locks: dict[str, asyncio.Lock] = {}
_pending: dict[str, list[BaseMessage]] = {}
def _get_lock(thread_id: str) -> asyncio.Lock:
if thread_id not in _locks:
_locks[thread_id] = asyncio.Lock()
return _locks[thread_id]
def extract_tool_call_count(messages: list[BaseMessage]) -> int:
"""Count total tool calls across all AI messages in the conversation."""
count = 0
for msg in messages:
tool_calls: Any = None
if hasattr(msg, "tool_calls"):
tool_calls = msg.tool_calls
elif isinstance(msg, dict):
tool_calls = msg.get("tool_calls")
if tool_calls:
count += len(tool_calls)
return count
def _format_messages_for_review(messages: list[BaseMessage], max_messages: int) -> str:
"""Serialize recent messages to a readable text block for the LLM reviewer."""
recent = messages[-max_messages:]
lines: list[str] = []
for msg in recent:
role = type(msg).__name__.replace("Message", "")
content = msg.content if isinstance(msg.content, str) else str(msg.content)
tool_calls = getattr(msg, "tool_calls", None)
if tool_calls:
calls_str = ", ".join(
f"{tc.name if hasattr(tc, 'name') else tc.get('name', '?')}()"
for tc in tool_calls
)
lines.append(f"[{role}] <tool_calls: {calls_str}>")
else:
preview = content[:400].replace("\n", " ")
lines.append(f"[{role}] {preview}")
return "\n".join(lines)
_REVIEW_SYSTEM_PROMPT = textwrap.dedent("""\
# Role: 技能评审代理(保守模式)
## Profile
- language: zh-CN
- description: 审阅已完成对话,仅在出现强证据的高价值可复用工作流时才沉淀为技能。**默认决策是 do-nothing** —— 宁可漏掉一个普通技能,也绝不创建垃圾技能。
- background: 历史问题:之前版本触发过于宽松,沉淀了大量低价值技能,污染了技能库。本版本针对性收紧。
- personality: 极度保守、苛刻挑剔、宁缺勿滥;面对模糊场景默认 do-nothing。
- expertise: 高价值工作流识别、严格去重、SKILL.md 规范、do-nothing 判定。
- target_audience: 技能库维护者、平台工程师。
## Skills
1. 工作流抽象与评审
- 价值评估: 区分一次性数据请求与具有沉淀价值的工作流。
- 去数据化归纳: 抽象出与具体数据无关的通用步骤、参数与注意事项。
- 决策映射: 在 do-nothing / patch / create 三者中做出可解释的唯一决定。
2. 文档与规范控制
- SKILL.md 编写: 简洁步骤 + 命令 + 坑点提示(非对话稿)。
- 命名规范: kebab-case(小写字母/数字/连字符,≤64 字符)。
- 触发描述: 以 "When" 开头的单句(≤1024 字符、无尖括号)。
- 结构校验: frontmatter 允许键、name 一致性、首尾分隔线。
3. 工具使用与错误处理
- skill_list 优先调用,识别功能重叠与现有覆盖范围。
- skill_manage 正确选 action=create / patch,并传递合规内容。
- 报错诊断与修正后才能重试,禁止内容不变重提。
4. 质量控制与安全
- 去重策略: 少而精,避免噪声与重复。
- 隐私合规: 严禁嵌入用户特定数据、文件路径、凭据。
- 一致性: name / description / content 三者语义对齐。
## Rules
### 0. 默认决策
**默认就是 do-nothing。** 仅当下面"必要条件"全部满足时才考虑创建;任一不满足 → do-nothing。垃圾技能比少一个技能更糟糕。
### 1. 创建必要条件(必须全部满足,AND 关系)
要创建一个新技能,以下条件**必须同时**满足:
1. **多步且有依赖**: 对话中至少 5 次工具调用,且步骤间存在依赖关系(不是并行查询,也不是同一工具的简单循环)。
2. **包含纠错经验**: 包含至少一次失败 / 报错 / 意外结果,agent 通过修正后达到目标。这种"踩坑—纠错"才是有沉淀价值的知识。
3. **跨场景可复用**: 抽离掉对话中的具体数据后,步骤本身在不同输入 / 不同数据集下仍然成立。
4. **不与现有技能重叠**: skill_list 中没有功能覆盖率 ≥70% 的现有技能(若有,应当 patch 而非新建)。
5. **能写出有料的 SKILL.md**: 抽象后能产出 ≥300 字、包含具体步骤 / 命令 / 坑点的工作流指南。写不出说明本来就没料。
### 2. 明确不要创建(即使工具调用很多也不要建)
- **单工具循环**: 连续多次 web_search 都只是搜索,没有形成工作流。
- **一次性数据查询**: 仅是"搜索 + 总结"模式,没有可复用结构。
- **LLM 常识可答**: 如"如何配置 Python 虚拟环境"这类公开文档 / 通识问题。
- **流程过度通用**: 如"先搜索再总结再回答"这种描述等于没说。
- **场景过窄**: 只在某一特定数据集 / 系统 / 项目下有效,换环境就失效。
- **内容字数不足**: 写不出 ≥300 字含真干货的 SKILL.md。
- **仅为"做点什么"而建**: 找不到清晰理由就 do-nothing,不要为了"完成 review"而强行创建。
### 3. 创建前自检(4 题,全部"是"才能 create)
提交 create 前在心中依次回答以下问题,任一为"否" → 立即放弃创建、改 do-nothing:
1. 如果我不创建这个技能,下一个遇到类似任务的用户会损失什么?(答得上来才证明有沉淀价值)
2. 这个技能是否能被一句话简短复述?若"能",说明信息量不足 → do-nothing。
3. SKILL.md 写完后是否包含至少 3 个具体步骤或命令?
4. 步骤中是否含有非显而易见的坑点提示(不是 LLM 凭常识就能猜到的)?
### 4. 单次 review 上限
**每次 review 最多创建 1 条新技能。** 即便发现多个候选,也只挑最有价值的那一个,其余 do-nothing。这是硬约束,不接受例外。
### 5. Patch 优先于 Create
若 skill_list 中已存在功能相近(即便不完全相同)的技能,优先选择 patch 现有技能补充改进点,而非新建。新技能名额只留给**全新类别**的工作流。
### 6. 格式与命名硬约束
- 命名:kebab-case,仅含小写字母 / 数字 / 连字符,≤64 字符(如 git-bisect-bug-hunt、api-load-test)。
- description:以 "When " 开头,单句,≤1024 字符,禁止尖括号 < >。
- content:合法 SKILL.md,包含 frontmatter 与正文。
- frontmatter 仅允许键:name、description、license、allowed-tools、metadata、compatibility、version、author。
- frontmatter 首行与末行必须为 "---" 独占一行。
- frontmatter 的 name 字段必须与 skill_manage 调用的 name 参数**完全一致**;不一致直接调用失败。
- 禁止添加 tags、category、type 等额外键。
- content 必须是简洁工作流指南,禁止对话转录。
- 严禁包含用户特定数据、文件路径、凭据。
### 7. 错误处理
- 调用 skill_manage 报错 → 阅读错误信息,定位根因,**修正内容后**再重试。
- 同名已存在 → 改用 action=patch(直接 create 同名会失败)。
- 禁止在未修改内容的情况下重复提交。
## Workflows
### 目标
仅在发现**高价值、可复用、有踩坑沉淀**的工作流时,创建或修补一条技能;其余一律 do-nothing。
### 步骤 1: 获取基线
调用 skill_list,记录现有技能名称与功能覆盖范围。这一步在每次 review 中必须执行。
### 步骤 2: 价值初筛(三关,任一不过即 do-nothing 结束)
通读对话并依次确认:
- 工具调用总数是否 ≥5?否则 do-nothing 结束。
- 是否包含失败 / 报错 / 修正过程?否则 do-nothing 结束。
- 步骤间是否存在依赖关系?否则 do-nothing 结束。
只有三关全过才进入步骤 3。
### 步骤 3: 候选抽象
将对话抽象为候选技能要素:
- 候选名称(kebab-case)
- 触发描述(When ... )
- 关键步骤、命令、坑点提示
- 剔除所有数据特定内容与隐私信息
### 步骤 4: 重叠检查
与 skill_list 比对:
- 完全无重叠 → 候选 action=create,进入步骤 5。
- 部分重叠(功能覆盖 ≥70%)→ 候选 action=patch,跳到步骤 6。
- 完全覆盖 → do-nothing 结束。
### 步骤 5: 创建前自检
若候选为 create,依次回答 Rules 第 3 节的 4 个自检问题。任一为"否" → 立即放弃创建,do-nothing 结束。全部为"是"才进入步骤 6。
### 步骤 6: 提交
- create:调用 skill_manage(action="create", name, content)。
- patch:调用 skill_manage(action="patch", name, content),仅针对差异 / 改进点。
- 一次 review 最多一次写操作。
### 步骤 7: 错误处理
若报错,按 Rules 第 7 节处理:阅读错误信息 → 修正根因 → 再试。禁止内容不变重提。
### 预期结果
- **多数情况下**: do-nothing。这是好的结果,不是失败。
- **少数情况下**: 一条高价值新技能,或对现有技能的一次有针对性 patch。
- **永远不要**: 为了"做点什么"而创建低价值技能。
## Initialization
作为技能评审代理(保守模式),你必须严格遵守 Rules,按 Workflows 执行。**默认决策是 do-nothing。** 只有强证据完全满足所有创建条件时,才考虑创建。创建一条垃圾技能比放弃一条普通技能更糟糕。
""")
async def _run_review(thread_id: str, messages: list[BaseMessage], app_config: "AppConfig", user_id: str = "default") -> None:
"""Core review logic: create_agent with full middleware chain, no 'silent creation' constraint.
Uses create_agent (the deerflow agent factory) directly so we can:
- Reuse the full middleware chain (ToolErrorHandlingMiddleware, etc.)
- Reuse the full tool set
- But pass app_config with skill_evolution.enabled=False to strip the
"Never create skills silently without user confirmation" constraint
from the system prompt.
Review instructions are injected via PromptPrefixMiddleware through
additional_kwargs["prompt_prefix"] on the HumanMessage.
"""
from langchain.agents import create_agent
from langchain_core.messages import HumanMessage
from langchain_core.runnables import RunnableConfig
from deerflow.agents.lead_agent.prompt import apply_prompt_template
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
from deerflow.agents.thread_state import ThreadState
from deerflow.models import create_chat_model
from deerflow.tools.tools import get_available_tools
evolution_cfg = app_config.skill_evolution
timeout = evolution_cfg.review_worker.timeout_seconds
max_messages = evolution_cfg.review_worker.max_messages
model_name = evolution_cfg.review_worker.model_name
conversation_text = _format_messages_for_review(messages, max_messages)
# Disable skill_evolution so the system prompt does NOT include
# "Never create skills silently without user confirmation".
# Also prevents SkillReviewMiddleware from registering (no recursive reviews).
review_app_config = app_config.model_copy(deep=True)
review_app_config.skill_evolution.enabled = False
try:
model = create_chat_model(name=model_name, app_config=review_app_config)
except Exception:
logger.warning("skill_review_worker: could not load model %r, aborting review", model_name)
return
tools = get_available_tools(app_config=review_app_config, subagent_enabled=False)
middlewares = build_lead_runtime_middlewares(app_config=review_app_config)
middlewares.append(PromptPrefixMiddleware())
system_prompt = apply_prompt_template(
subagent_enabled=False,
max_concurrent_subagents=0,
app_config=review_app_config,
memory_injection_enabled=False,
)
agent = create_agent(
model=model,
tools=tools,
middleware=middlewares,
system_prompt=system_prompt,
state_schema=ThreadState,
)
# RunnableConfig supplies thread_id so ThreadDataMiddleware is satisfied.
import uuid
run_config = RunnableConfig(configurable={"thread_id": f"__skill-review__{uuid.uuid4()}"})
# Review instructions arrive as prompt_prefix: PromptPrefixMiddleware prepends
# them before every model call without persisting them to state.
first_message = HumanMessage(
content=(
"Review this conversation and apply any skill changes you deem appropriate.\n\n"
f"<conversation>\n{conversation_text}\n</conversation>"
),
additional_kwargs={"prompt_prefix": _REVIEW_SYSTEM_PROMPT},
)
def _invoke_with_user_context() -> dict:
from deerflow.runtime.user_context import set_current_user, reset_current_user
class _SimpleUser:
def __init__(self, uid: str) -> None:
self.id = uid
token = set_current_user(_SimpleUser(user_id))
try:
return agent.invoke({"messages": [first_message]}, config=run_config)
finally:
reset_current_user(token)
try:
result = await asyncio.wait_for(
asyncio.get_event_loop().run_in_executor(None, _invoke_with_user_context),
timeout=timeout,
)
# Log the final AI message so we can see what the review agent decided.
final_msgs = (result or {}).get("messages", [])
final_text = ""
for msg in reversed(final_msgs):
content = getattr(msg, "content", None)
if content and isinstance(content, str) and content.strip():
final_text = content.strip()[:1000]
break
logger.info(
"skill_review_worker: review completed for thread %s — agent said: %s",
thread_id,
final_text or "(no text response)",
)
except TimeoutError:
logger.warning("skill_review_worker: review timed out after %ds (thread %s)", timeout, thread_id)
except Exception:
logger.exception("skill_review_worker: review failed for thread %s", thread_id)
async def _run_review_with_pending(
thread_id: str,
messages: list[BaseMessage],
app_config: "AppConfig",
user_id: str = "default",
) -> None:
"""Run one review; if a newer snapshot arrived while we ran, drain it."""
lock = _get_lock(thread_id)
async with lock:
await _run_review(thread_id, messages, app_config, user_id)
pending = _pending.pop(thread_id, None)
if pending is not None:
asyncio.create_task(_run_review_with_pending(thread_id, pending, app_config, user_id))
def maybe_start_review(
thread_id: str,
messages: list[BaseMessage],
app_config: "AppConfig",
user_id: str = "default",
) -> bool:
"""Schedule a background review for ``thread_id``.
Returns True if a new task was started, False if queued or skipped.
"""
evolution_cfg = app_config.skill_evolution
if not evolution_cfg.enabled or not evolution_cfg.review_worker.enabled:
return False
tool_count = extract_tool_call_count(messages)
if tool_count < evolution_cfg.review_worker.min_tool_calls:
logger.debug(
"skill_review_worker: skipping review for thread %s (%d tool calls < min %d)",
thread_id,
tool_count,
evolution_cfg.review_worker.min_tool_calls,
)
return False
lock = _get_lock(thread_id)
if lock.locked():
_pending[thread_id] = messages
logger.debug("skill_review_worker: queued review for thread %s (review in progress)", thread_id)
return False
asyncio.create_task(_run_review_with_pending(thread_id, messages, app_config, user_id))
logger.info("skill_review_worker: started review for thread %s (%d tool calls) user=%s", thread_id, tool_count, user_id)
return True
@@ -1,7 +1,7 @@
"""笔记本 RAG 检索工具。
"""空间 RAG 检索工具。
从 LangGraph configurable 中读取 notebook_search_space_id / notebook_source_ids /
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在笔记本模式下引用回答。
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在空间模式下引用回答。
API base URL 通过环境变量 DEER_FLOW_NOTEBOOK_API_BASE 配置。
"""
@@ -134,27 +134,58 @@ def _default_agent_store() -> Any | None:
return None
async def _assert_agent_may_delete(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
"""Pre-flight ownership check before agent-delete touches the on-disk file.
async def _assert_skill_is_agent_authored(name: str) -> None:
"""Refuse to modify skills authored by a human (UI upload / adopt / legacy).
LocalSkillStorage doesn't know about users — it deletes anything that
matches by name. Without this guard, a malicious or hallucinating agent
could destroy another user's skill just by knowing the name. We refuse
unless the DB confirms ``user_id`` owns the skill outright.
Even when ownership lines up, we draw a hard line at content the *human*
curated through the Web UI: uploads, adopted orphan skills, and legacy
skills with no usage entry (which default to ``source='upload'``).
Those represent reviewed content the user has explicitly endorsed —
letting the agent rewrite them risks silent drift away from what was
approved. Only skills marked ``source='agent'`` (set by
``mark_agent_created`` in the ``create`` flow) remain editable so the
evolution loop can keep refining them.
Returns the DB record for use in the subsequent cascade step. Raises
``ValueError`` (mapped to a tool error visible to the LLM) on refusal.
Raises ``ValueError`` (mapped to a tool error visible to the LLM) on
refusal.
"""
from deerflow.skills.usage import get_entry
entry = await _to_thread(get_entry, name)
if entry.get("source") != "agent":
raise ValueError(
f"Refused: skill '{name}' was authored by a human via the Web UI "
f"and cannot be modified by an agent. Ask the owner to edit it "
f"through the Web UI, or have the user duplicate it to a new "
f"agent-owned skill first."
)
async def _assert_agent_may_modify(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
"""Pre-flight ownership check before an agent-driven write touches a custom skill.
LocalSkillStorage operates by name only — patch/edit/write_file/remove_file
and delete all resolve a name to a directory without knowing who owns it.
Without this guard a malicious or hallucinating agent could overwrite or
destroy another user's skill (including UI-uploaded skills) just by
knowing the name. We refuse unless the DB confirms ``user_id`` owns the
skill outright.
Returns the DB record so callers that need owner info (e.g. delete
cascade) can reuse it. Raises ``ValueError`` (mapped to a tool error
visible to the LLM) on refusal.
"""
record = await store.get_any(name)
if record is None:
# No DB row — could be a legacy/orphan skill. Refuse rather than
# delete blindly; admin can clean up via the Web UI.
# touch blindly; users should adopt it via /api/skills/reconcile-custom
# or an admin can clean it up via the Web UI.
raise ValueError(
f"Refused: skill '{name}' has no ownership record. Ask a human to delete it via the Web UI."
f"Refused: skill '{name}' has no ownership record. Ask a human to adopt or manage it via the Web UI."
)
owner = record.get("owner_user_id")
if owner is None:
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be deleted by an agent.")
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be modified by an agent.")
if owner != user_id:
raise ValueError(f"Refused: skill '{name}' is owned by another user.")
return record
@@ -200,6 +231,10 @@ async def _skill_manage_impl(
lock = _get_lock(name)
thread_id = _get_thread_id(runtime)
skill_storage = get_or_new_skill_storage()
# Resolved up-front so ownership checks in the write-action branches stay
# one-liners. Cheap when the DB is unconfigured (returns None).
user_id = get_effective_user_id()
skill_store = _default_skill_store()
async with lock:
if action == "create":
@@ -224,14 +259,16 @@ async def _skill_manage_impl(
# surfaces it (custom skills without a DB row are filtered out by
# ``GET /api/skills``). Resolve the user_id *here* on the main
# async stack — contextvars don't propagate into ``to_thread``.
store = _default_skill_store()
if store is not None:
await _ensure_skill_db_registered(store, name, get_effective_user_id())
if skill_store is not None:
await _ensure_skill_db_registered(skill_store, name, user_id)
await refresh_skills_system_prompt_cache_async()
return f"Created custom skill '{name}'."
if action == "edit":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if content is None:
raise ValueError("content is required for edit.")
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
@@ -250,6 +287,9 @@ async def _skill_manage_impl(
if action == "patch":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if find is None or replace is None:
raise ValueError("find and replace are required for patch.")
skill_file = skill_storage.get_custom_skill_file(name)
@@ -277,11 +317,10 @@ async def _skill_manage_impl(
# Ownership pre-check: LocalSkillStorage deletes purely by name,
# so without this an agent could blow away another user's skill
# just by knowing the name. Raise BEFORE touching the filesystem.
user_id = get_effective_user_id()
store = _default_skill_store()
record: dict[str, Any] | None = None
if store is not None:
record = await _assert_agent_may_delete(store, name, user_id)
if skill_store is not None:
record = await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
await _to_thread(
skill_storage.delete_custom_skill,
@@ -306,9 +345,9 @@ async def _skill_manage_impl(
await _to_thread(forget, name)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
if store is not None:
if skill_store is not None:
try:
await store.delete(name, user_id)
await skill_store.delete(name, user_id)
except Exception:
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
agent_store = _default_agent_store()
@@ -330,6 +369,9 @@ async def _skill_manage_impl(
if action == "write_file":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if path is None or content is None:
raise ValueError("path and content are required for write_file.")
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
@@ -347,6 +389,9 @@ async def _skill_manage_impl(
if action == "remove_file":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if path is None:
raise ValueError("path is required for remove_file.")
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)