Files
xuhd be757b1f2f @
feat(skills): skill_list 可见性过滤 + skill 只读工具常驻 + 进化面板对照展示

- skill_list:默认 lead agent 改为只暴露未归档、且内置或属于当前用户的技能;
  自定义 agent 仍严格按白名单。SkillStore 不可用时降级放行全部自定义技能。
- prompt:注入系统提示词的技能列表剔除已归档(state=archived)技能。
- tools:skill_list/skill_view 只读工具改为常驻提供;skill_manage 写工具仍按
  生效配置(含 per-user 覆盖)的 skill_evolution.enabled 门控。
- curator:should_run_now 解析异常时改为不运行;get_curator_status 返回上次
  LLM 建议与实际工具调用两份记录,供前端对照。
- 前端:技能进化面板新增建议 vs 实际动作对照展示 + 配套 i18n。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
@
2026-05-15 20:39:13 +08:00

962 lines
40 KiB
Python

"""Skill curator — automatic lifecycle transitions and LLM consolidation pass.
Per-user since the multitenancy upgrade: every public function takes a
``user_id`` argument and the on-disk state lives under
``{base_dir}/users/{user_id}/.curator_state`` (alongside the per-user
memory.json and threads/ dirs).
"""
from __future__ import annotations
import asyncio
import concurrent.futures
import json
import logging
import os
import re
import tempfile
import threading
from collections.abc import Coroutine
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, TypeVar
logger = logging.getLogger(__name__)
_CURATOR_STATE_FILENAME = ".curator_state"
_T = TypeVar("_T")
def _run_async_blocking(coro: Coroutine[Any, Any, _T]) -> _T:
"""Run an async coroutine to completion from sync curator code.
Curator is invoked from a background thread, but ``agent.invoke()`` inside
``_run_llm_review`` (LangChain 1.x ``create_agent``) spins up its own
asyncio loop which can leave the calling thread in an "already has a
running loop" state. In that case raw ``asyncio.run(coro)`` raises
``RuntimeError`` *after* the coroutine object has been materialised but
before anyone awaits it — producing the ``coroutine was never awaited``
RuntimeWarning that pollutes our logs.
This helper detects a running loop in the caller's thread and offloads
to a dedicated worker thread; otherwise it falls back to ``asyncio.run``.
Either way it guarantees the coroutine is closed on the failure path,
so no RuntimeWarning ever escapes.
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
except BaseException:
# ``asyncio.run`` ran in the worker (it consumes the coroutine on
# success or surfaces the inner exception); but if the executor
# itself failed to submit, close to be safe.
try:
coro.close()
except Exception:
pass
raise
try:
return asyncio.run(coro)
except BaseException:
try:
coro.close()
except Exception:
pass
raise
def _get_skills_root() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path())
def _curator_state_path(user_id: str | None) -> Path:
"""Return the state file path. Per-user when user_id is known; falls back
to the global ``skills/.curator_state`` for legacy callers."""
if not user_id:
return _get_skills_root() / _CURATOR_STATE_FILENAME
from deerflow.config.paths import get_paths
return get_paths().user_dir(user_id) / _CURATOR_STATE_FILENAME
def _read_curator_state(user_id: str | None) -> dict:
path = _curator_state_path(user_id)
if not path.exists():
return {"run_count": 0, "paused": False}
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {"run_count": 0, "paused": False}
def _write_curator_state(state: dict, user_id: str | None) -> None:
path = _curator_state_path(user_id)
try:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
json.dump(state, f, indent=2, default=str)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except Exception:
pass
raise
except Exception:
logger.debug("Failed to write curator state", exc_info=True)
def _get_curator_config(user_id: str | None = None):
"""Return the curator config object for ``user_id``.
Reads ``config.yaml.skill_evolution.curator`` as the baseline and merges
the per-user override file (``users/{user_id}/skill_evolution.json``) on
top. With ``user_id=None`` falls back to the global default.
"""
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
skill_evo = load_skill_evolution_config_for_user(user_id)
return getattr(skill_evo, "curator", None)
except Exception:
return None
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
"""Return skill names owned by ``user_id`` (used to filter transitions).
Returns None if the SkillStore is unreachable — curator then degrades to
"process every on-disk agent-created skill" (legacy behaviour).
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
async def _run() -> list[dict]:
return await store.list_visible(user_id)
records = _run_async_blocking(_run())
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("_list_user_owned_skill_names failed", exc_info=True)
return None
def _collect_referenced_skill_names() -> set[str]:
"""Scan every agent's config.yaml and return the union of their ``skills``
fields.
Used as the "do not touch" set during curator transitions and LLM
candidate selection — any skill currently bound to *any* agent
(including the owner's own) is exempt from automatic stale/archive
and from being suggested for merging.
Returns an empty set on filesystem errors (best-effort, non-fatal).
"""
import yaml
from deerflow.config.paths import get_paths
referenced: set[str] = set()
try:
agents_dir = get_paths().agents_dir
except Exception:
return referenced
if not agents_dir.exists():
return referenced
for entry in agents_dir.iterdir():
if not entry.is_dir():
continue
config_file = entry / "config.yaml"
if not config_file.exists():
continue
try:
with config_file.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
except Exception:
logger.debug("_collect_referenced_skill_names: failed to parse %s", config_file, exc_info=True)
continue
skills = data.get("skills")
if isinstance(skills, list):
for s in skills:
if isinstance(s, str):
referenced.add(s)
return referenced
def _is_skill_published(skill_name: str) -> bool:
"""Return True if ``skill_name`` is marked ``published=True`` in SkillStore.
Best-effort — returns False on any error so a transient DB hiccup never
causes a published skill to be wrongly archived (the caller treats
"published=True" as exempt, so False = "fail open to processing", which
only puts a skill *back* into curator scope when DB is unavailable; in
that degraded mode curator legacy behaviour applies).
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return False
store = make_skill_store(factory)
async def _run() -> dict | None:
return await store.get_any(skill_name)
record = _run_async_blocking(_run())
return bool(record and record.get("published", False))
except Exception:
logger.debug("_is_skill_published(%s) failed", skill_name, exc_info=True)
return False
def _eligible_for_curation(
name: str,
entry: dict,
*,
owned_set: set[str] | None,
referenced_set: set[str],
) -> bool:
"""Three-condition gate for whether curator may touch a skill.
Curator only acts on skills that ALL of:
1. source == "agent" (caller is expected to have already filtered, but
we double-check here for safety)
2. NOT referenced by any agent's skills config
3. NOT published (queried via SkillStore)
``owned_set`` is the set of skill names owned by the running user. When
None, the legacy "process every agent-created skill" fallback applies
for ownership only (the other two conditions still hold).
"""
if entry.get("source") != "agent":
return False
if owned_set is not None and name not in owned_set:
return False
if name in referenced_set:
return False
if _is_skill_published(name):
return False
return True
def should_run_now(user_id: str) -> bool:
"""Return True if the curator interval has elapsed for ``user_id`` and it
is not paused."""
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
return False
state = _read_curator_state(user_id)
if state.get("paused"):
return False
last_run = state.get("last_run_at")
if not last_run:
return True
interval_hours = getattr(curator_cfg, "interval_hours", 168)
try:
last_dt = datetime.fromisoformat(last_run)
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
except Exception:
return False
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
"""Pure age-based state transitions — no LLM involved.
Only processes agent-created skills that are ALL of:
- owned by ``user_id``
- not referenced by any agent's ``skills`` config
- not currently published
Skips pinned skills. Returns counters: {marked_stale, archived, checked,
skipped_referenced, skipped_published}.
"""
from deerflow.skills.usage import STATE_ACTIVE, STATE_ARCHIVED, STATE_STALE, agent_created_entries, latest_activity_at, set_state
if now is None:
now = datetime.now(UTC)
curator_cfg = _get_curator_config(user_id)
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
raw_entries = agent_created_entries()
skipped_referenced = 0
skipped_published = 0
eligible: dict[str, dict] = {}
for name, entry in raw_entries.items():
if owned is not None and name not in owned:
continue
if name in referenced:
skipped_referenced += 1
continue
if _is_skill_published(name):
skipped_published += 1
continue
eligible[name] = entry
counters = {
"marked_stale": 0,
"archived": 0,
"checked": len(eligible),
"skipped_referenced": skipped_referenced,
"skipped_published": skipped_published,
}
from deerflow.skills import curator_log
for name, entry in eligible.items():
if entry.get("pinned"):
continue
last_activity_str = latest_activity_at(name)
if not last_activity_str:
continue
try:
last_dt = datetime.fromisoformat(last_activity_str)
if last_dt.tzinfo is None:
last_dt = last_dt.replace(tzinfo=UTC)
except Exception:
continue
days_idle = (now - last_dt).days
current_state = entry.get("state", STATE_ACTIVE)
if days_idle >= archive_days and current_state != STATE_ARCHIVED:
set_state(name, STATE_ARCHIVED)
counters["archived"] += 1
curator_log.record(
"info",
f" 自动归档 [{name}]:已闲置 {days_idle} 天(阈值 {archive_days} 天)",
user_id=user_id,
)
elif days_idle >= stale_days and current_state == STATE_ACTIVE:
set_state(name, STATE_STALE)
counters["marked_stale"] += 1
curator_log.record(
"info",
f" 标记过时 [{name}]:已闲置 {days_idle} 天(阈值 {stale_days} 天)",
user_id=user_id,
)
return counters
def _render_candidate_list(user_id: str) -> str:
"""List of skill candidates handed to the LLM for merge/archive review.
Same three-condition gate as ``apply_automatic_transitions``: only
agent-created, not-referenced-by-any-agent, not-published skills owned
by ``user_id`` show up here. The LLM literally cannot see (and thus
cannot suggest action on) anything outside that set.
"""
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
entries: dict[str, dict] = {}
for k, v in all_entries().items():
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
continue
if v.get("state") == STATE_ARCHIVED:
continue
entries[k] = v
if not entries:
return "No agent-created active skills found."
lines = []
for name, entry in sorted(entries.items()):
state = entry.get("state", "active")
use = entry.get("use_count", 0)
view = entry.get("view_count", 0)
patch = entry.get("patch_count", 0)
last = entry.get("last_used_at") or entry.get("created_at") or "unknown"
lines.append(f"- {name} [state={state}, use={use}, view={view}, patches={patch}, last_active={last}]")
return "\n".join(lines)
_CURATOR_REVIEW_PROMPT = (
"You are running as DeerFlow's background skill CURATOR. This is an "
"UMBRELLA-BUILDING consolidation pass, not a passive audit and not a "
"duplicate-finder.\n\n"
"The goal of the skill collection is a LIBRARY OF CLASS-LEVEL "
"INSTRUCTIONS AND EXPERIENTIAL KNOWLEDGE. A collection of hundreds of "
"narrow skills where each one captures one session's specific bug is "
"a FAILURE of the library — not a feature. An agent searching skills "
"matches on descriptions, not on exact names; one broad umbrella "
"skill with labeled subsections beats five narrow siblings for "
"discoverability, not the other way around.\n\n"
"The right target shape is CLASS-LEVEL skills with rich SKILL.md "
"bodies + `references/`, `templates/`, and `scripts/` subfiles for "
"session-specific detail — not one-session-one-skill micro-entries.\n\n"
"Hard rules — do not violate:\n"
"1. DO NOT touch public (built-in) skills. The candidate list below "
"is already filtered to agent-created custom skills only.\n"
"2. DO NOT delete any skill. Archiving (a soft state flip via the "
"skill_archive tool — the file stays in custom/<name>/ and the user "
"can restore it from the Web UI) is the maximum action you may take. "
"Archives are recoverable; deletion is not. "
"When you call skill_archive(name, reason) you MUST pass a `reason` "
"string written **in Chinese**, in plain language a human operator "
"would read — NOT English jargon or terse debug notes. Examples of "
"acceptable reasons: '已合并到 code-formatter,该技能同时覆盖 Python 与 JS', "
"'与 foo-bar 内容重复,保留 foo-bar', '重构后过时,被 api-v2-search 替代', "
"'一次性 POC 实验,无复用价值'. Bad reasons (do NOT use): 'merged', "
"'duplicate', 'obsolete', 'consolidated'. The reason is shown in the "
"Web UI run-history for user audit, so write it as you would explain "
"it to the user themselves.\n"
"3. DO NOT touch skills shown as pinned=yes. Skip them entirely.\n"
"4. DO NOT use usage counters as a reason to skip consolidation. The "
"counters are new and often mostly zero. Judge overlap on CONTENT, "
"not on use_count. 'use=0' is not evidence a skill is valuable; it's "
"absence of evidence either way.\n"
"5. DO NOT reject consolidation on the grounds that 'each skill has "
"a distinct trigger'. Pairwise distinctness is the wrong bar. The "
"right bar is: 'would a human maintainer write this as N separate "
"skills, or as one skill with N labeled subsections?' When the "
"answer is the latter, merge.\n\n"
"How to work — not optional:\n"
"1. Scan the full candidate list. Identify PREFIX CLUSTERS (skills "
"sharing a first word or domain keyword). Examples you are likely "
"to find: search-*, code-*, data-*, api-*, file-*, analysis-*, "
"workflow-*, report-*, monitor-*, debug-*, python-*, security-*, etc. "
"Expect 10-25 clusters.\n"
"2. For each cluster with 2+ members, do NOT ask 'are these pairs "
"overlapping?' — ask 'what is the UMBRELLA CLASS these skills all "
"serve? Would a maintainer name that class and write one skill for "
"it?' If yes, pick (or create) the umbrella and absorb the siblings "
"into it.\n"
"3. Three ways to consolidate — use the right one per cluster:\n"
" a. MERGE INTO EXISTING UMBRELLA — one skill in the cluster is "
"already broad enough to be the umbrella. Patch it to add a labeled "
"section for each sibling's unique insight, then archive the "
"siblings.\n"
" b. CREATE A NEW UMBRELLA SKILL.md — no existing member is broad "
"enough. Create a new class-level skill whose SKILL.md covers the "
"shared workflow and has short labeled subsections. Archive the "
"now-absorbed narrow siblings.\n"
" c. DEMOTE TO REFERENCES/TEMPLATES/SCRIPTS — a sibling has "
"narrow-but-valuable session-specific content. Move it into the "
"umbrella's appropriate support directory:\n"
" • `references/<topic>.md` for session-specific detail OR "
"condensed knowledge banks (quoted research, API docs excerpts, "
"domain notes, provider quirks, reproduction recipes)\n"
" • `templates/<name>.<ext>` for starter files meant to be "
"copied and modified\n"
" • `scripts/<name>.<ext>` for statically re-runnable actions "
"(verification scripts, fixture generators, probes)\n"
" Then archive the old sibling.\n"
"4. Also flag skills whose NAME is too narrow (contains a specific "
"issue number, a feature codename, a specific error string, an "
"'audit' / 'diagnosis' / 'salvage' session artifact). These almost "
"always belong as a subsection or support file under a class-level "
"umbrella.\n"
"5. Iterate. After one consolidation round, scan the remaining set "
"and look for the NEXT umbrella opportunity. Don't stop after 3 "
"merges.\n\n"
"'keep' is a legitimate decision ONLY when the skill is already a "
"class-level umbrella and none of the proposed merges would improve "
"discoverability. 'This is narrow but distinct from its siblings' "
"is NOT a reason to keep — it's a reason to move it under an "
"umbrella as a subsection or support file.\n\n"
"Expected output: real umbrella-ification. Process every obvious "
"cluster. If you end the pass with fewer than 10 archives, you "
"stopped too early — go back and look at the clusters you left "
"alone.\n\n"
"When done, write a human summary AND a structured machine-readable "
"block so downstream tooling can distinguish consolidation from "
"pruning. Format EXACTLY:\n\n"
"## Structured summary (required)\n"
"```yaml\n"
"consolidations:\n"
" - from: <old-skill-name>\n"
" into: <umbrella-skill-name>\n"
" reason: <one short sentence — why merged, not just 'similar'>\n"
"prunings:\n"
" - name: <skill-name>\n"
" reason: <one short sentence — why archived with no merge target>\n"
"```\n\n"
"Every skill you archived via skill_archive MUST appear in exactly one "
"of the two lists. If you consolidated X into umbrella Y (patched Y, wrote "
"a references file to Y, or created Y with X's content absorbed), X "
"goes under `consolidations` with `into: Y`. If you archived X with "
"no absorption — truly stale, irrelevant, or obsolete — X goes under "
"`prunings`. Leave a list empty (`consolidations: []`) if none. Do "
"not omit the block. The block comes AFTER your human-readable "
"summary of clusters processed, patches made, and decisions left alone."
)
def _make_skill_archive_tool(user_id: str):
"""Curator-only tool: soft-archive a skill (sets state, files stay put)."""
from langchain.tools import tool as lc_tool
from deerflow.skills import curator_log
from deerflow.skills.storage import get_or_new_skill_storage
@lc_tool("skill_archive")
def _skill_archive(name: str, reason: str) -> str:
"""Soft-archive an agent-created skill (sets state=archived; files stay in custom/).
The user can recover the skill from the Web UI's archive drawer.
This is intentionally non-destructive — the on-disk SKILL.md and any
supporting files are preserved so nothing is lost.
Args:
name: The skill name to archive.
reason: 必填。用**中文**写一句话,清晰说明为什么归档这个技能 —
直接给读它的运维人员看,不要写英文术语 / 调试信息。例如:
- "已合并到 code-formatter,该技能同时覆盖 Python 与 JS"
- "重构后过时,被 api-v2-search 替代"
- "与现有 foo-bar 技能重复,内容完全一致"
- "一次性 POC 实验,无复用价值"
- "命名过窄(含具体 issue 号),已并入通用伞技能 X"
此原因会展示在策展日志和 Web UI 运行历史中,供用户审计。
"""
from deerflow.skills.usage import STATE_ARCHIVED, get_entry, set_state
# Defensive: tolerate empty/None reason but log a clear placeholder
# rather than crashing the curator run.
reason_text = (reason or "").strip() or "未提供原因"
storage = get_or_new_skill_storage()
if not storage.custom_skill_exists(name):
curator_log.record("warning", f"归档:未找到技能 [{name}],或它已被归档", user_id=user_id)
return f"Skill '{name}' not found or already archived."
entry = get_entry(name)
if entry.get("pinned"):
curator_log.record("info", f"归档:技能 [{name}] 已被钉选,跳过", user_id=user_id)
return f"Skill '{name}' is pinned — skipping."
if entry.get("state") == STATE_ARCHIVED:
curator_log.record("info", f"归档:技能 [{name}] 已是归档状态,跳过", user_id=user_id)
return f"Skill '{name}' is already archived."
# Safety net: even if the LLM hallucinated a name not in the
# candidate list, refuse to archive anything that is published or
# currently referenced by an agent.
if _is_skill_published(name):
curator_log.record(
"warning",
f"归档:已拒绝处理 [{name}],该技能已发布(策展不允许操作已发布的技能)",
user_id=user_id,
)
return f"Refused: skill '{name}' is published; manual unpublish is required first."
referenced = _collect_referenced_skill_names()
if name in referenced:
curator_log.record(
"warning",
f"归档:已拒绝处理 [{name}],该技能正被某个 Agent 引用",
user_id=user_id,
)
return f"Refused: skill '{name}' is still referenced by one or more agents."
# Soft archive: flip the lifecycle state. The skill directory stays in
# ``custom/<name>/`` untouched so the user can recover it from the
# Web UI's archive drawer (which lists skills by state, not by location).
try:
set_state(name, STATE_ARCHIVED)
except Exception as e:
curator_log.record("error", f"归档失败:技能 [{name}] — {e}", user_id=user_id)
return f"Archive failed: {e}"
curator_log.record(
"info",
f"已归档(软):[{name}] — 原因:{reason_text} — 文件保留在 custom/{name}/",
user_id=user_id,
)
return f"Archived '{name}' (reason: {reason_text}; files preserved in custom/{name}/)."
return _skill_archive
def _make_curator_skill_manage_tool():
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
"""
from langchain.tools import tool as lc_tool
from deerflow.tools.skill_manage_tool import _skill_manage_impl
@lc_tool("skill_manage")
def _curator_skill_manage(
action: str,
name: str,
content: str | None = None,
path: str | None = None,
find: str | None = None,
replace: str | None = None,
expected_count: int | None = None,
) -> str:
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
支持 action:create / patch / edit / write_file / remove_file / list / view。
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
Args:
action: 操作类型(见上;delete 会被拒绝)。
name: 连字符命名的技能名。
content: create / edit / write_file 的新文件内容。
path: write_file / remove_file 的支持文件路径。
find: patch 要替换的原文本。
replace: patch 的替换文本。
expected_count: patch 的预期替换次数(可选)。
"""
if action == "delete":
return (
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
)
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
return _run_async_blocking(
_skill_manage_impl(
runtime=None,
action=action,
name=name,
content=content,
path=path,
find=find,
replace=replace,
expected_count=expected_count,
)
)
return _curator_skill_manage
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
"""Run the curator consolidation pass as a React agent with skill tools bound.
返回 ``(llm_text, tool_actions)``:
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
"""
from deerflow.skills import curator_log
try:
from langchain.agents import create_agent
from langchain_core.messages import HumanMessage
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.models import create_chat_model
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
model = create_chat_model()
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
# (wrong action / missing required args), the exception is converted
# into a ToolMessage(status="error") so the LLM can read the error and
# retry, instead of aborting the entire review pass.
# NOTE: ``create_react_agent`` (langgraph.prebuilt) does NOT accept a
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
messages = result.get("messages", [])
tool_actions = _extract_tool_actions(messages)
for msg in reversed(messages):
if not getattr(msg, "tool_calls", None):
content = getattr(msg, "content", "")
if isinstance(content, str) and content.strip():
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return content, tool_actions
if isinstance(content, list):
text = " ".join(
block.get("text", "") if isinstance(block, dict) else str(block)
for block in content
).strip()
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return text, tool_actions
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
return "", tool_actions
except Exception as e:
logger.warning("Curator LLM review failed: %s", e)
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
return "", []
def _parse_structured_summary(text: str) -> dict:
pattern = re.compile(r"```yaml\s*(.*?)```", re.DOTALL)
match = pattern.search(text)
if not match:
return {"consolidations": [], "prunings": []}
try:
import yaml
data = yaml.safe_load(match.group(1)) or {}
return {"consolidations": data.get("consolidations") or [], "prunings": data.get("prunings") or []}
except Exception:
return {"consolidations": [], "prunings": []}
def _extract_tool_actions(messages: list) -> list[dict]:
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
是否一致。每条形如 ``{tool, args, status, result}``。
"""
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
calls: dict[str, dict] = {}
for msg in messages:
for tc in getattr(msg, "tool_calls", None) or []:
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not tc_id:
continue
calls[tc_id] = {
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
}
# 再按 ToolMessage(带 tool_call_id)收集真实结果
actions: list[dict] = []
for msg in messages:
tc_id = getattr(msg, "tool_call_id", None)
if not tc_id:
continue
call = calls.get(tc_id, {})
raw = getattr(msg, "content", "")
result = raw if isinstance(raw, str) else str(raw)
actions.append(
{
"tool": call.get("tool") or getattr(msg, "name", None),
"args": call.get("args", {}),
"status": getattr(msg, "status", None) or "success",
"result": result[:500],
}
)
return actions
def run_curator_review(user_id: str) -> dict:
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
Only skills owned by ``user_id`` are touched.
"""
from deerflow.skills import curator_log
curator_log.start_new_run(user_id=user_id)
now = datetime.now(UTC)
logger.info("Curator[%s]: starting run at %s", user_id, now.isoformat())
curator_log.record("info", f"本次策展开始,启动时间 {now.isoformat()}", user_id=user_id)
transition_counters = apply_automatic_transitions(user_id, now)
logger.info("Curator[%s]: transitions=%s", user_id, transition_counters)
curator_log.record(
"info",
(
f"按年龄自动流转完成:已检查 {transition_counters['checked']} 个,"
f"标记陈旧 {transition_counters['marked_stale']} 个,"
f"自动归档 {transition_counters['archived']} 个"
),
user_id=user_id,
)
candidate_list = _render_candidate_list(user_id)
candidate_line_count = 0 if not candidate_list.strip() else len([line for line in candidate_list.splitlines() if line.startswith("- ")])
curator_log.record(
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
)
if candidate_line_count == 0:
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
suggestions = {"consolidations": [], "prunings": []}
tool_actions: list[dict] = []
else:
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
# Surface per-item reasoning so operators can audit *why* each action
# was taken, not just the count.
for item in suggestions.get("consolidations", []) or []:
if not isinstance(item, dict):
continue
name = item.get("from", "?")
into = item.get("into", "?")
reason = (item.get("reason") or "无说明").strip()
curator_log.record(
"info",
f" 合并建议 [{name}] → [{into}]:{reason}",
user_id=user_id,
)
for item in suggestions.get("prunings", []) or []:
if not isinstance(item, dict):
continue
name = item.get("name", "?")
reason = (item.get("reason") or "无说明").strip()
curator_log.record(
"info",
f" 删减建议 [{name}]:{reason}",
user_id=user_id,
)
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
for act in tool_actions:
target = act.get("args", {}).get("name", "")
curator_log.record(
"info",
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
user_id=user_id,
)
state = _read_curator_state(user_id)
state["last_run_at"] = now.isoformat()
state["run_count"] = state.get("run_count", 0) + 1
state["last_run_summary"] = (
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
f"归档 {transition_counters['archived']} 个;"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
)
state["last_consolidation_suggestions"] = suggestions # LLM 自述
state["last_actual_actions"] = tool_actions # 真实工具调用记录
_write_curator_state(state, user_id)
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
curator_log.record("info", f"本次策展已完成 — {state['last_run_summary']}", user_id=user_id)
return {"transitions": transition_counters, "suggestions": suggestions}
class _CuratorUser:
"""Minimal ``CurrentUser`` protocol implementation for background runs.
Curator threads are spawned via ``threading.Thread`` (not an asyncio task),
so the ``deerflow.runtime.user_context`` ContextVar starts unset. Without
a stand-in, any tool the LLM review invokes (e.g. ``skill_manage(create)``)
would call ``get_effective_user_id()`` and silently fall back to the
``DEFAULT_USER_ID`` sentinel — causing newly-created skills to be owned
by ``"default"`` instead of the user the curator is running for.
"""
__slots__ = ("id",)
def __init__(self, user_id: str) -> None:
self.id = user_id
def _curator_thread_target(user_id: str) -> None:
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.skills import curator_log
# Bind the per-thread user context for the entire curator run so every
# downstream tool / repository call resolves the correct user_id.
token = set_current_user(_CuratorUser(user_id))
try:
run_curator_review(user_id)
except Exception as exc:
logger.exception("Curator[%s] background run failed", user_id)
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
finally:
reset_current_user(token)
async def maybe_run_curator() -> None:
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
任何失败都不会阻断调用方。
"""
try:
from sqlalchemy import select
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.user.model import UserRow
factory = get_session_factory()
if factory is None:
return
async with factory() as session:
result = await session.execute(select(UserRow.id))
user_ids = [str(row[0]) for row in result.all()]
except Exception:
logger.exception("Curator: failed to enumerate users at startup")
return
for user_id in user_ids:
try:
if not should_run_now(user_id):
continue
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
daemon=True,
name=f"skill-curator-{user_id}",
)
thread.start()
except Exception:
logger.exception("Curator: failed to spawn for user %s", user_id)
def get_curator_status(user_id: str) -> dict:
"""Return curator state for ``user_id``."""
from deerflow.skills.usage import STATE_ACTIVE, STATE_ARCHIVED, STATE_STALE, agent_created_entries
state = _read_curator_state(user_id)
owned = _list_user_owned_skill_names(user_id)
entries = agent_created_entries()
if owned is not None:
entries = {name: entry for name, entry in entries.items() if name in owned}
counts: dict[str, int] = {STATE_ACTIVE: 0, STATE_STALE: 0, STATE_ARCHIVED: 0}
for entry in entries.values():
s = entry.get("state", STATE_ACTIVE)
counts[s] = counts.get(s, 0) + 1
cfg = _get_curator_config(user_id)
return {
"enabled": cfg is not None and getattr(cfg, "enabled", True),
"run_count": state.get("run_count", 0),
"last_run_at": state.get("last_run_at"),
"last_run_summary": state.get("last_run_summary"),
"paused": state.get("paused", False),
"agent_created_skills": {
"total": len(entries),
"active": counts[STATE_ACTIVE],
"stale": counts[STATE_STALE],
"archived": counts[STATE_ARCHIVED],
},
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
or {"consolidations": [], "prunings": []},
"last_actual_actions": state.get("last_actual_actions") or [],
}