This commit is contained in:
xuhd committed 2026-05-16 15:51:49 +08:00
commit 48ddbf0412
423 files changed
+117115 -3214

No files matched your search

@@ -0,0 +1,3 @@
from deerflow.agents.canvas_agent.agent import make_canvas_agent
__all__ = ["make_canvas_agent"]
@@ -0,0 +1,68 @@
from __future__ import annotations
import logging
from langchain.agents import create_agent
from langchain_core.runnables import RunnableConfig
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
from deerflow.agents.canvas_agent.tools import update_artifact_tool
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.agents.thread_state import ThreadState
from deerflow.config.app_config import get_app_config
from deerflow.models import create_chat_model
from deerflow.tools.builtins import ask_clarification_tool
logger = logging.getLogger(__name__)
def _get_runtime_config(config: RunnableConfig) -> dict:
cfg = dict(config.get("configurable", {}) or {})
context = config.get("context", {}) or {}
if isinstance(context, dict):
cfg.update(context)
return cfg
def make_canvas_agent(config: RunnableConfig):
"""LangGraph graph factory for canvas mode; signature compatible with LangGraph Server."""
app_config = get_app_config()
cfg = _get_runtime_config(config)
requested_model = cfg.get("model_name") or cfg.get("model")
if requested_model and app_config.get_model_config(requested_model):
model_name = requested_model
else:
if requested_model:
logger.warning("Canvas agent: model '%s' not found; falling back to default.", requested_model)
model_name = app_config.models[0].name if app_config.models else None
if model_name is None:
raise ValueError("No chat models configured. Please add at least one model in config.yaml.")
thinking_enabled = cfg.get("thinking_enabled", False)
model_config = app_config.get_model_config(model_name)
if thinking_enabled and model_config and not model_config.supports_thinking:
thinking_enabled = False
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
middlewares = [
DanglingToolCallMiddleware(),
ToolErrorHandlingMiddleware(),
TitleMiddleware(app_config=app_config),
LoopDetectionMiddleware(),
ClarificationMiddleware(),
]
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
tools=[update_artifact_tool, ask_clarification_tool],
middleware=middlewares,
system_prompt=CANVAS_SYSTEM_PROMPT,
state_schema=ThreadState,
)
@@ -0,0 +1,40 @@
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
<canvas_behavior>
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
ALWAYS call `update_artifact` to save your work. Never embed the artifact content in your chat response.
Rules:
- On every request that creates or changes the artifact, call `update_artifact` ONCE with the COMPLETE updated content.
- In your text reply, describe what you did in 1–2 sentences.
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
</canvas_behavior>
<artifact_context>
The current artifact state is injected into your context. When editing, build on the existing content exactly.
When creating from scratch, produce a complete, high-quality artifact in one shot.
</artifact_context>
<selection_rewrite>
When the user's message contains a <selection> block, you MUST:
1. Identify the exact selection in the current artifact content.
2. Rewrite ONLY that selection according to the user's instruction.
3. Keep the rest of the artifact exactly unchanged.
4. Call update_artifact with the complete artifact where the selection has been replaced.
Example: if the artifact is "A B C" and the selection is "B" with instruction "make it bold",
produce "A **B** C" — not a new document.
</selection_rewrite>
<quick_actions>
When the user triggers a quick action, apply it to the full artifact unless specific highlighted text is provided:
- expand: Add depth and detail while preserving structure
- shorten: Condense to the essential points
- formalize: Rewrite in professional, formal language
- casualize: Rewrite in friendly, conversational language
- add_comments: Add clear, helpful code comments
- add_logs: Add appropriate logging statements
- fix_bugs: Identify and fix all bugs in the code
- optimize_code: Improve performance and readability
</quick_actions>
"""
@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import Annotated, Literal
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
from langchain_core.messages import ToolMessage
from langgraph.types import Command
from langgraph.typing import ContextT
from deerflow.agents.thread_state import CanvasArtifact, CanvasArtifactCode, CanvasArtifactText, ThreadState
@tool("update_artifact", parse_docstring=True)
def update_artifact_tool(
runtime: ToolRuntime[ContextT, ThreadState],
tool_call_id: Annotated[str, InjectedToolCallId],
type: Literal["text", "code"],
title: str,
content: str,
language: str = "other",
) -> Command:
"""Create or update the canvas artifact. Always call this to persist your work.
Args:
type: Artifact type — "text" for markdown documents, "code" for source code.
title: Human-readable title of the artifact.
content: Full artifact content: markdown text or complete source code.
language: Programming language (for code type only, e.g. "python", "typescript"). Ignored for text.
"""
current: CanvasArtifact | None = (runtime.state or {}).get("artifact") if runtime.state else None
if current is not None:
existing_contents = list(current.get("contents", []))
new_index = current.get("currentIndex", len(existing_contents)) + 1
else:
existing_contents = []
new_index = 1
if type == "text":
new_entry: CanvasArtifactText | CanvasArtifactCode = CanvasArtifactText(
index=new_index,
type="text",
title=title,
fullMarkdown=content,
)
else:
new_entry = CanvasArtifactCode(
index=new_index,
type="code",
title=title,
language=language,
code=content,
)
updated: CanvasArtifact = {
"currentIndex": new_index,
"contents": existing_contents + [new_entry],
}
return Command(
update={
"artifact": updated,
"messages": [ToolMessage("Artifact updated.", tool_call_id=tool_call_id)],
}
)
@@ -8,6 +8,7 @@ from deerflow.agents.lead_agent.prompt import apply_prompt_template
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
@@ -255,6 +256,10 @@ def _build_middlewares(
resolved_app_config = app_config or get_app_config()
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
# Inject the admin prompt prefix into the model request (kept out of the
# persisted/displayed message so the user never sees it).
middlewares.append(PromptPrefixMiddleware())
# Add summarization middleware if enabled
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
if summarization_middleware is not None:
@@ -331,6 +336,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
subagent_enabled = cfg.get("subagent_enabled", False)
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
writing_mode = bool(cfg.get("writing_mode", False))
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
writing_artifact_path = None
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
@@ -350,7 +359,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
thinking_enabled = False
logger.info(
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s",
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s",
agent_id or "default",
thinking_enabled,
reasoning_effort,
@@ -358,6 +367,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
is_plan_mode,
subagent_enabled,
max_concurrent_subagents,
writing_mode,
)
# Inject run metadata for LangSmith trace tagging
@@ -382,6 +392,8 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
"reasoning_effort": reasoning_effort,
"is_plan_mode": is_plan_mode,
"subagent_enabled": subagent_enabled,
"writing_mode": writing_mode,
"writing_artifact_path": writing_artifact_path,
"tool_groups": agent_config.tool_groups if agent_config else None,
"available_skills": runtime_available_skills,
}
@@ -402,6 +414,8 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
system_prompt=apply_prompt_template(
subagent_enabled=subagent_enabled,
max_concurrent_subagents=max_concurrent_subagents,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
available_skills=set(["bootstrap"]),
app_config=resolved_app_config,
),
@@ -424,6 +438,8 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
max_concurrent_subagents=max_concurrent_subagents,
agent_id=agent_id,
agent_name=agent_display_name,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
),
@@ -69,8 +69,25 @@ _enabled_skills_refresh_version = 0
_enabled_skills_refresh_event = threading.Event()
def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
"""从注入提示词的技能列表里剔除已归档(usage state=archived)的技能。
技能归档是软状态:SKILL.md 仍留在 custom/<name>/、enabled 标志不变,所以
load_skills(enabled_only=True) 仍会返回它。但归档技能不应再出现在 agent 的系统
提示词里 —— 对 agent 而言它应等同于"不存在"。失败时不过滤(返回原列表)。
"""
try:
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
entries = all_entries()
return [s for s in skills if entries.get(s.name, {}).get("state") != STATE_ARCHIVED]
except Exception:
logger.debug("Failed to filter archived skills for prompt injection", exc_info=True)
return skills
def _load_enabled_skills_sync() -> list[Skill]:
return list(get_or_new_skill_storage().load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage().load_skills(enabled_only=True)))
def _start_enabled_skills_refresh_thread() -> None:
@@ -171,7 +188,7 @@ def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[
"""
if app_config is None:
return _get_enabled_skills()
return list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True)))
def _skill_mutability_label(category: SkillCategory | str) -> str:
@@ -481,6 +498,8 @@ You: "Deploying to staging..." [proceed]
{deferred_tools_section}
{writing_mode_section}
{subagent_section}
<working_directory existed="true">
@@ -806,6 +825,8 @@ def apply_prompt_template(
*,
agent_id: str | None = None,
agent_name: str | None = None,
writing_mode: bool = False,
writing_artifact_path: str | None = None,
available_skills: set[str] | None = None,
app_config: AppConfig | None = None,
) -> str:
@@ -841,6 +862,32 @@ def apply_prompt_template(
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
current_writing_artifact = (
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
if writing_artifact_path
else ""
)
writing_mode_section = (
f"""
<writing_mode>
Writing mode is enabled for this run.
**Mandatory Output Contract**
- Treat the user's request as a writing/report-generation task.
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
{current_writing_artifact}
</writing_mode>
"""
if writing_mode
else ""
)
# Build ACP agent section only if ACP agents are configured
acp_section = _build_acp_section(app_config=app_config)
@@ -856,6 +903,7 @@ def apply_prompt_template(
soul=get_agent_soul(storage_agent_id),
skills_section=skills_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
network_policy_section=network_policy_section,
subagent_section=subagent_section,
@@ -0,0 +1,78 @@
"""Middleware that injects the admin-configured prompt prefix at model-call time.
The prompt-prefix feature lets an admin prepend a fixed instruction to the
user's question before it reaches the LLM. To keep the prefix invisible in the
chat UI, the Gateway does **not** concatenate it into the persisted
``HumanMessage.content``; instead it tags the first human message with
``additional_kwargs["prompt_prefix"]`` (see ``app.gateway.services``).
This middleware reads that tag and prepends the prefix to the message content
*only in the request handed to the model*. The persisted state — and therefore
what the frontend renders — keeps the clean, prefix-free content.
"""
from collections.abc import Awaitable, Callable
from typing import override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import HumanMessage
PROMPT_PREFIX_KEY = "prompt_prefix"
class PromptPrefixMiddleware(AgentMiddleware[AgentState]):
"""Prepends ``additional_kwargs['prompt_prefix']`` to human messages for the LLM only.
The transformation is applied to the model request copy, never written back
to state, so it is naturally idempotent across the multiple model calls of a
single turn.
"""
@staticmethod
def _apply(messages: list) -> list | None:
"""Return a patched copy with prefixes prepended, or None if nothing to do."""
patched: list | None = None
for idx, msg in enumerate(messages):
if not isinstance(msg, HumanMessage):
continue
raw_prefix = (getattr(msg, "additional_kwargs", None) or {}).get(PROMPT_PREFIX_KEY)
prefix = str(raw_prefix).strip() if raw_prefix else ""
if not prefix:
continue
content = msg.content
if isinstance(content, str):
new_content: object = f"{prefix}\n\n{content}".strip("\n")
elif isinstance(content, list):
# Structured content (text blocks + media): prepend a text block.
new_content = [{"type": "text", "text": prefix}, *content]
else:
new_content = f"{prefix}\n\n{content!s}"
if patched is None:
patched = list(messages)
# model_copy preserves the message id so streaming dedup is unaffected.
patched[idx] = msg.model_copy(update={"content": new_content})
return patched
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
patched = self._apply(request.messages)
if patched is not None:
request = request.override(messages=patched)
return handler(request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
patched = self._apply(request.messages)
if patched is not None:
request = request.override(messages=patched)
return await handler(request)
@@ -72,7 +72,7 @@ class SetupAgentApprovalMiddleware(AgentMiddleware[SetupAgentApprovalMiddlewareS
approval_msg = ToolMessage(
id=self._stable_message_id(tool_call_id, args),
content="请确认是否创建/更新这个专家配置。",
content="请确认是否创建/更新这个智能体配置。",
tool_call_id=tool_call_id,
name="setup_agent",
additional_kwargs={"setup_agent_approval": payload},
@@ -1,4 +1,4 @@
from typing import Annotated, NotRequired, TypedDict
from typing import Annotated, Literal, NotRequired, Optional, TypedDict
from langchain.agents import AgentState
@@ -18,6 +18,26 @@ class ViewedImageData(TypedDict):
mime_type: str
class CanvasArtifactText(TypedDict):
index: int
type: Literal["text"]
title: str
fullMarkdown: str
class CanvasArtifactCode(TypedDict):
index: int
type: Literal["code"]
title: str
language: str
code: str
class CanvasArtifact(TypedDict):
currentIndex: int
contents: list[CanvasArtifactText | CanvasArtifactCode]
def merge_artifacts(existing: list[str] | None, new: list[str] | None) -> list[str]:
"""Reducer for artifacts list - merges and deduplicates artifacts."""
if existing is None:
@@ -53,3 +73,11 @@ class ThreadState(AgentState):
todos: NotRequired[list | None]
uploaded_files: NotRequired[list[dict] | None]
viewed_images: Annotated[dict[str, ViewedImageData], merge_viewed_images] # image_path -> {base64, mime_type}
# Canvas fields (coexist with artifacts: list[str] which tracks file outputs)
artifact: NotRequired[Optional[CanvasArtifact]]
highlighted_text: NotRequired[Optional[str]]
highlighted_code: NotRequired[Optional[str]]
canvas_mode: NotRequired[Optional[Literal["text", "code"]]]
quick_action_id: NotRequired[Optional[str]]
web_search_enabled: NotRequired[bool]
web_search_results: NotRequired[Optional[list[dict]]]
@@ -4,10 +4,17 @@ from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
@@ -17,10 +24,6 @@ class SkillEvolutionConfig(BaseModel):
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
moderation_model_name: str | None = Field(
default=None,
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
@@ -0,0 +1,60 @@
"""Create tags and tag_assignments tables.
Revision ID: 20260515_01
Revises: 20260514_02
Create Date: 2026-05-15
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260515_01"
down_revision = "20260514_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("tags"):
op.create_table(
"tags",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("tag_type", sa.String(32), nullable=False),
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),
)
op.create_index("ix_tags_tag_type", "tags", ["tag_type"])
if not _has_table("tag_assignments"):
op.create_table(
"tag_assignments",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("tag_id", sa.String(64), nullable=False),
sa.Column("target_type", sa.String(32), nullable=False),
sa.Column("target_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
)
op.create_index("ix_tag_assignments_tag_id", "tag_assignments", ["tag_id"])
op.create_index("ix_tag_assignments_target_type", "tag_assignments", ["target_type"])
op.create_index("ix_tag_assignments_target_id", "tag_assignments", ["target_id"])
def downgrade() -> None:
if _has_table("tag_assignments"):
op.drop_table("tag_assignments")
if _has_table("tags"):
op.drop_table("tags")
@@ -30,6 +30,7 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.user.model import UserRow
@@ -47,6 +48,8 @@ __all__ = [
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SkillRow",
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"UserRow",
]
@@ -67,3 +67,13 @@ class SkillStore(ABC):
async def count_owned(self, user_id: str) -> int:
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def list_published_names(self) -> set[str]:
"""Return the set of skill names with ``published = True`` (any owner).
Batch alternative to calling ``get_any`` per skill — the curator uses
this to filter out published skills in a single query instead of one
DB round-trip per candidate skill.
"""
raise NotImplementedError
@@ -127,3 +127,6 @@ class MemorySkillStore(SkillStore):
async def count_owned(self, user_id: str) -> int:
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
async def list_published_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("published")}
@@ -142,3 +142,9 @@ class SkillRepository(SkillStore):
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def list_published_names(self) -> set[str]:
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
async with self._sf() as session:
result = await session.execute(stmt)
return {row[0] for row in result.all()}
@@ -0,0 +1,14 @@
"""Persistence layer for tags and tag-to-resource assignments."""
from deerflow.persistence.tags.base import TagStore
from deerflow.persistence.tags.memory import MemoryTagStore
from deerflow.persistence.tags.model import TAG_TYPES
from deerflow.persistence.tags.sql import TagRepository
__all__ = ["TagStore", "MemoryTagStore", "TagRepository", "TAG_TYPES", "make_tag_store"]
def make_tag_store(session_factory):
if session_factory is None:
return MemoryTagStore()
return TagRepository(session_factory)
@@ -0,0 +1,73 @@
"""Abstract persistence API for tags and tag-to-resource assignments."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class TagStore(ABC):
# ------------------------------------------------------------------ tags
@abstractmethod
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
"""List tags, optionally filtered by ``tag_type``.
Each record includes a derived ``usage_count`` (number of assignments).
"""
raise NotImplementedError
@abstractmethod
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create a tag. ``data`` must carry ``id``, ``name``, ``tag_type``.
Raises ``ValueError`` if ``(tag_type, name)`` is not unique.
"""
raise NotImplementedError
@abstractmethod
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update ``name`` / ``color`` / ``description``. ``tag_type`` is immutable.
Returns ``None`` if the tag does not exist. Raises ``ValueError`` on a
name collision within the type.
"""
raise NotImplementedError
@abstractmethod
async def delete_tag(self, tag_id: str) -> bool:
"""Delete a tag and cascade-remove all of its assignments."""
raise NotImplementedError
# ----------------------------------------------------------- assignments
@abstractmethod
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
"""Return the tags currently assigned to one resource."""
raise NotImplementedError
@abstractmethod
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
"""Replace the full set of tags on a resource.
Every id in ``tag_ids`` must reference an existing tag whose
``tag_type`` equals ``target_type``; otherwise raises ``ValueError``.
"""
raise NotImplementedError
@abstractmethod
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
"""Bulk-fetch tags for many resources at once (avoids N+1 on list pages).
Returns ``{target_id: [tag, ...]}`` for ids that have at least one tag.
"""
raise NotImplementedError
@abstractmethod
async def unassign_all(self, target_type: str, target_id: str) -> int:
"""Drop every tag assignment for a resource. Returns the count removed."""
raise NotImplementedError
@@ -0,0 +1,138 @@
"""In-memory ``TagStore`` implementation for tests."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.tags.base import TagStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(record: dict[str, Any]) -> dict[str, Any]:
return copy.deepcopy(record)
class MemoryTagStore(TagStore):
def __init__(self) -> None:
self._tags: dict[str, dict[str, Any]] = {}
# assignments keyed by surrogate id
self._assignments: dict[str, dict[str, Any]] = {}
# ------------------------------------------------------------------ tags
def _usage_count(self, tag_id: str) -> int:
return sum(1 for a in self._assignments.values() if a["tag_id"] == tag_id)
def _with_count(self, record: dict[str, Any]) -> dict[str, Any]:
out = _copy(record)
out["usage_count"] = self._usage_count(record["id"])
return out
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
rows = [r for r in self._tags.values() if tag_type is None or r["tag_type"] == tag_type]
rows.sort(key=lambda r: r["created_at"])
return [self._with_count(r) for r in rows]
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
record = self._tags.get(tag_id)
return self._with_count(record) if record is not None else None
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
tag_id = data["id"]
name = data["name"]
tag_type = data["tag_type"]
if any(r["tag_type"] == tag_type and r["name"] == name for r in self._tags.values()):
raise ValueError(f"Tag '{name}' already exists for type '{tag_type}'")
record = {
"id": tag_id,
"name": name,
"tag_type": tag_type,
"color": data.get("color") or "gray",
"description": data.get("description") or "",
"created_at": _now(),
"updated_at": _now(),
}
self._tags[tag_id] = record
return self._with_count(record)
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
record = self._tags.get(tag_id)
if record is None:
return None
new_name = data.get("name", record["name"])
if any(r["id"] != tag_id and r["tag_type"] == record["tag_type"] and r["name"] == new_name for r in self._tags.values()):
raise ValueError(f"Tag '{new_name}' already exists for type '{record['tag_type']}'")
if "name" in data:
record["name"] = data["name"]
if "color" in data:
record["color"] = data["color"]
if "description" in data:
record["description"] = data["description"]
record["updated_at"] = _now()
return self._with_count(record)
async def delete_tag(self, tag_id: str) -> bool:
if tag_id not in self._tags:
return False
del self._tags[tag_id]
for aid in [aid for aid, a in self._assignments.items() if a["tag_id"] == tag_id]:
del self._assignments[aid]
return True
# ----------------------------------------------------------- assignments
def _tags_for(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
tag_ids = [a["tag_id"] for a in self._assignments.values() if a["target_type"] == target_type and a["target_id"] == target_id]
records = [self._tags[tid] for tid in tag_ids if tid in self._tags]
records.sort(key=lambda r: r["name"])
return [self._with_count(r) for r in records]
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
return self._tags_for(target_type, target_id)
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
unique_ids = list(dict.fromkeys(tag_ids))
for tid in unique_ids:
tag = self._tags.get(tid)
if tag is None:
raise ValueError(f"Tag '{tid}' does not exist")
if tag["tag_type"] != target_type:
raise ValueError(f"Tag '{tid}' is type '{tag['tag_type']}', cannot tag a '{target_type}' resource")
# drop existing, then re-add
for aid in [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]:
del self._assignments[aid]
for tid in unique_ids:
aid = f"{tid}:{target_type}:{target_id}"
self._assignments[aid] = {
"id": aid,
"tag_id": tid,
"target_type": target_type,
"target_id": target_id,
"created_at": _now(),
}
return self._tags_for(target_type, target_id)
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
wanted = set(target_ids)
out: dict[str, list[dict[str, Any]]] = {}
for a in self._assignments.values():
if a["target_type"] != target_type or a["target_id"] not in wanted:
continue
tag = self._tags.get(a["tag_id"])
if tag is None:
continue
out.setdefault(a["target_id"], []).append(self._with_count(tag))
for tags in out.values():
tags.sort(key=lambda r: r["name"])
return out
async def unassign_all(self, target_type: str, target_id: str) -> int:
victims = [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]
for aid in victims:
del self._assignments[aid]
return len(victims)
@@ -0,0 +1,47 @@
"""ORM models for tags and tag-to-resource assignments.
Two tables:
- ``tags`` — a tag belongs to exactly one ``tag_type`` (``agent`` / ``skill`` /
``scheduled_task``). Tag names are unique *within* a type.
- ``tag_assignments`` — many-to-many link between a tag and a target resource.
``target_type`` is denormalized from the tag so the table is queryable by
``(target_type, target_id)`` without a join. The API layer enforces the
invariant ``target_type == tag.tag_type``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
# Allowed tag/target types. Kept here so persistence + API share one source.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
class TagRow(Base):
__tablename__ = "tags"
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class TagAssignmentRow(Base):
__tablename__ = "tag_assignments"
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,187 @@
"""SQLAlchemy-backed tag + tag-assignment repository."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tags.base import TagStore
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
def _tag_to_dict(row: TagRow, usage_count: int = 0) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
data["usage_count"] = usage_count
return data
class TagRepository(TagStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
async def _usage_counts(session: AsyncSession, tag_ids: list[str]) -> dict[str, int]:
if not tag_ids:
return {}
stmt = select(TagAssignmentRow.tag_id, func.count()).where(TagAssignmentRow.tag_id.in_(tag_ids)).group_by(TagAssignmentRow.tag_id)
result = await session.execute(stmt)
return {tag_id: count for tag_id, count in result.all()}
# ------------------------------------------------------------------ tags
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
stmt = select(TagRow)
if tag_type is not None:
stmt = stmt.where(TagRow.tag_type == tag_type)
stmt = stmt.order_by(TagRow.created_at.asc())
async with self._sf() as session:
rows = list((await session.execute(stmt)).scalars())
counts = await self._usage_counts(session, [r.id for r in rows])
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return None
counts = await self._usage_counts(session, [tag_id])
return _tag_to_dict(row, counts.get(tag_id, 0))
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = TagRow(
id=data["id"],
name=data["name"],
tag_type=data["tag_type"],
color=data.get("color") or "gray",
description=data.get("description") or "",
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise ValueError(f"Tag '{data['name']}' already exists for type '{data['tag_type']}'") from exc
await session.refresh(row)
return _tag_to_dict(row, 0)
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return None
if "name" in data:
row.name = data["name"]
if "color" in data:
row.color = data["color"]
if "description" in data:
row.description = data["description"]
row.updated_at = datetime.now(UTC)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise ValueError(f"Tag '{data.get('name')}' already exists for type '{row.tag_type}'") from exc
await session.refresh(row)
counts = await self._usage_counts(session, [tag_id])
return _tag_to_dict(row, counts.get(tag_id, 0))
async def delete_tag(self, tag_id: str) -> bool:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return False
await session.execute(delete(TagAssignmentRow).where(TagAssignmentRow.tag_id == tag_id))
await session.delete(row)
await session.commit()
return True
# ----------------------------------------------------------- assignments
async def _tags_for(self, session: AsyncSession, target_type: str, target_id: str) -> list[dict[str, Any]]:
stmt = (
select(TagRow)
.join(TagAssignmentRow, TagAssignmentRow.tag_id == TagRow.id)
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id == target_id)
.order_by(TagRow.name.asc())
)
rows = list((await session.execute(stmt)).scalars())
counts = await self._usage_counts(session, [r.id for r in rows])
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
return await self._tags_for(session, target_type, target_id)
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
unique_ids = list(dict.fromkeys(tag_ids))
async with self._sf() as session:
if unique_ids:
found = list((await session.execute(select(TagRow).where(TagRow.id.in_(unique_ids)))).scalars())
by_id = {r.id: r for r in found}
for tid in unique_ids:
tag = by_id.get(tid)
if tag is None:
raise ValueError(f"Tag '{tid}' does not exist")
if tag.tag_type != target_type:
raise ValueError(f"Tag '{tid}' is type '{tag.tag_type}', cannot tag a '{target_type}' resource")
await session.execute(
delete(TagAssignmentRow).where(
TagAssignmentRow.target_type == target_type,
TagAssignmentRow.target_id == target_id,
)
)
now = datetime.now(UTC)
for tid in unique_ids:
session.add(
TagAssignmentRow(
id=uuid.uuid4().hex,
tag_id=tid,
target_type=target_type,
target_id=target_id,
created_at=now,
)
)
await session.commit()
return await self._tags_for(session, target_type, target_id)
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
if not target_ids:
return {}
stmt = (
select(TagAssignmentRow.target_id, TagRow)
.join(TagRow, TagAssignmentRow.tag_id == TagRow.id)
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id.in_(target_ids))
.order_by(TagRow.name.asc())
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
tag_rows = [tag for _, tag in rows]
counts = await self._usage_counts(session, [t.id for t in tag_rows])
out: dict[str, list[dict[str, Any]]] = {}
for target_id, tag in rows:
out.setdefault(target_id, []).append(_tag_to_dict(tag, counts.get(tag.id, 0)))
return out
async def unassign_all(self, target_type: str, target_id: str) -> int:
async with self._sf() as session:
result = await session.execute(
delete(TagAssignmentRow).where(
TagAssignmentRow.target_type == target_type,
TagAssignmentRow.target_id == target_id,
)
)
await session.commit()
return result.rowcount or 0
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import concurrent.futures
import contextvars
import json
import logging
import os
@@ -25,6 +26,10 @@ logger = logging.getLogger(__name__)
_CURATOR_STATE_FILENAME = ".curator_state"
# 周期调度一轮里同时执行的 curator 数量上限。到期用户多时(尤其首次部署所有用户一起
# 到期)用线程池排队消费,避免一次性起 N 个线程齐调 LLM 打爆 provider 速率限制。
_CURATOR_MAX_CONCURRENCY = 5
_T = TypeVar("_T")
@@ -119,6 +124,81 @@ def _write_curator_state(state: dict, user_id: str | None) -> None:
logger.debug("Failed to write curator state", exc_info=True)
def _parse_state_time(value: str | None) -> datetime | None:
"""解析 ``.curator_state`` 里的 ISO 时间字符串,返回 aware datetime(UTC)。
naive(无时区)字符串按 UTC 补全 —— 与 ``apply_automatic_transitions`` 的处理
一致,避免 naive 时间串与 ``datetime.now(UTC)`` 直接比较时抛异常。无法解析时
返回 None,由调用方决定降级行为。
"""
if not value:
return None
try:
dt = datetime.fromisoformat(value)
except Exception:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=UTC)
return dt
# Curator 运行锁:防止同一用户的 curator 并发执行(定时器 vs 手动触发、连点手动触发)。
# 锁状态写进 .curator_state 的 running/started_at 字段,跨 worker 可见(手动请求可能
# 落到非调度 worker)。started_at 超过下面的阈值视为进程残留,允许接管以免永久死锁。
_CURATOR_STALE_LOCK_HOURS = 6
# 上一轮"尝试"过但未成功(last_run_at 始终为空)时的重试间隔 —— 比正常 interval_hours
# 短得多,但也不至于每个调度 tick(1h)都重试、每次都烧一份 LLM 配额。
_CURATOR_RETRY_HOURS = 6
# 进程内串行化运行锁的"读-改-写",保证同进程内的抢锁是原子的(跨进程的极小竞态窗口
# 可接受 —— curator 触发频率极低)。
_run_lock_guard = threading.Lock()
def _run_lock_active(state: dict, now: datetime) -> bool:
"""判断 ``.curator_state`` 里的运行锁当前是否有效(running 且未过期)。
``started_at`` 缺失或不可解析、或距今已超过 ``_CURATOR_STALE_LOCK_HOURS`` 时,
视为进程残留留下的死锁,返回 False 以便后续触发能接管。
"""
if not state.get("running"):
return False
started_dt = _parse_state_time(state.get("started_at"))
if started_dt is None:
return False
return now < started_dt + timedelta(hours=_CURATOR_STALE_LOCK_HOURS)
def _try_acquire_run_lock(user_id: str, now: datetime) -> bool:
"""尝试为 ``user_id`` 抢占 curator 运行锁。
抢到锁(已把 running=True / started_at 写回)返回 True;已有未过期的运行时返回
False。读-改-写在 ``_run_lock_guard`` 下进行,保证同进程内原子。
"""
with _run_lock_guard:
state = _read_curator_state(user_id)
if _run_lock_active(state, now):
return False
state["running"] = True
state["started_at"] = now.isoformat()
_write_curator_state(state, user_id)
return True
def _release_run_lock(user_id: str) -> None:
"""释放 curator 运行锁并记录本次尝试时间。无论运行成功或异常都应调用。
除了把 running 置回 False,还写入 ``last_attempt_at`` —— 用于失败退避:首次运行
若一直崩溃,``last_run_at`` 永远为空,``should_run_now`` 改用
``last_attempt_at + _CURATOR_RETRY_HOURS`` 判定,避免每个 tick 都重试。
成功跑完后 ``last_run_at`` 已被写入,``should_run_now`` 不再看 ``last_attempt_at``。
"""
with _run_lock_guard:
state = _read_curator_state(user_id)
state["running"] = False
state["last_attempt_at"] = datetime.now(UTC).isoformat()
_write_curator_state(state, user_id)
def _get_curator_config(user_id: str | None = None):
"""Return the curator config object for ``user_id``.
@@ -135,6 +215,21 @@ def _get_curator_config(user_id: str | None = None):
return None
def _skill_evolution_enabled(user_id: str | None = None) -> bool:
"""返回技能进化总开关 ``skill_evolution.enabled``(已合并 per-user 覆盖)。
这是整套技能进化功能的总闸:关闭时 curator 完全不动作。任何读取异常都按
"关闭"处理(fail-safe)—— 宁可不跑,也不在配置不可读时偷偷运行。
"""
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
skill_evo = load_skill_evolution_config_for_user(user_id)
return bool(getattr(skill_evo, "enabled", False))
except Exception:
return False
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
"""Return skill names owned by ``user_id`` (used to filter transitions).
@@ -231,12 +326,39 @@ def _is_skill_published(skill_name: str) -> bool:
return False
def _list_published_skill_names() -> set[str]:
"""一次性返回所有 ``published=True`` 的技能名集合。
取代"每个技能调一次 ``_is_skill_published``"的逐条查库 —— curator 一轮里会遍历
所有候选技能,逐条查会产生 ≈2N 次独立 event loop + DB 往返。出错时返回空集
(fail-open:与 ``_is_skill_published`` 出错返回 False 语义一致,即 DB 不可用时
所有技能都视为未发布,回退到 curator 的 legacy 行为)。
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return set()
store = make_skill_store(factory)
async def _run() -> set[str]:
return await store.list_published_names()
return _run_async_blocking(_run())
except Exception:
logger.debug("_list_published_skill_names failed", exc_info=True)
return set()
def _eligible_for_curation(
name: str,
entry: dict,
*,
owned_set: set[str] | None,
referenced_set: set[str],
published_set: set[str],
) -> bool:
"""Three-condition gate for whether curator may touch a skill.
@@ -244,11 +366,15 @@ def _eligible_for_curation(
1. source == "agent" (caller is expected to have already filtered, but
we double-check here for safety)
2. NOT referenced by any agent's skills config
3. NOT published (queried via SkillStore)
3. NOT published
``owned_set`` is the set of skill names owned by the running user. When
None, the legacy "process every agent-created skill" fallback applies
for ownership only (the other two conditions still hold).
``published_set`` is the set of published skill names, pre-fetched once by
the caller via ``_list_published_skill_names()`` — avoids a per-skill DB
round-trip.
"""
if entry.get("source") != "agent":
return False
@@ -256,7 +382,7 @@ def _eligible_for_curation(
return False
if name in referenced_set:
return False
if _is_skill_published(name):
if name in published_set:
return False
return True
@@ -264,21 +390,52 @@ def _eligible_for_curation(
def should_run_now(user_id: str) -> bool:
"""Return True if the curator interval has elapsed for ``user_id`` and it
is not paused."""
curator_cfg = _get_curator_config()
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
# 总开关:技能进化整体关闭时,curator 一律不跑。
if not _skill_evolution_enabled(user_id):
return False
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
return False
state = _read_curator_state(user_id)
if state.get("paused"):
return False
# 运行锁:已有 curator 在跑(且锁未过期)时不再触发新的一轮。
if _run_lock_active(state, datetime.now(UTC)):
return False
last_run = state.get("last_run_at")
if not last_run:
return True
# 从未成功跑完过。若此前已尝试过(last_attempt_at 存在),按较短的重试间隔
# 退避 —— 避免首次运行持续失败时每个 tick(1h)都重试、每次烧一份 LLM 配额。
attempt_dt = _parse_state_time(state.get("last_attempt_at"))
if attempt_dt is None:
return True
return datetime.now(UTC) >= attempt_dt + timedelta(hours=_CURATOR_RETRY_HOURS)
interval_hours = getattr(curator_cfg, "interval_hours", 168)
try:
last_dt = datetime.fromisoformat(last_run)
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
except Exception:
last_dt = _parse_state_time(last_run)
if last_dt is None:
# 时间串损坏 —— 视同"从未跑过",允许重跑并重写正确时间戳,而不是像旧逻辑那样
# except 返回 False 导致该用户永久不再触发。
return True
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
def manual_trigger_blocked_reason(user_id: str) -> str | None:
"""手动触发 curator 的预检。返回 None 表示可以跑;否则返回阻塞原因代码:
``evolution_disabled`` / ``curator_disabled`` / ``already_running``。
供 ``POST /api/curator/run`` 在派发后台线程**前**同步判定 —— 后台线程
(``_curator_thread_target``)本身也会再查一遍,这里只是为了让前端拿到准确反馈,
而不是无论开关状态如何都提示"已开始"。
"""
if not _skill_evolution_enabled(user_id):
return "evolution_disabled"
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
return "curator_disabled"
state = _read_curator_state(user_id)
if _run_lock_active(state, datetime.now(UTC)):
return "already_running"
return None
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
@@ -297,12 +454,13 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
if now is None:
now = datetime.now(UTC)
curator_cfg = _get_curator_config()
curator_cfg = _get_curator_config(user_id)
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
raw_entries = agent_created_entries()
skipped_referenced = 0
@@ -314,7 +472,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
if name in referenced:
skipped_referenced += 1
continue
if _is_skill_published(name):
if name in published:
skipped_published += 1
continue
eligible[name] = entry
@@ -377,9 +535,10 @@ def _render_candidate_list(user_id: str) -> str:
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
entries: dict[str, dict] = {}
for k, v in all_entries().items():
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced, published_set=published):
continue
if v.get("state") == STATE_ARCHIVED:
continue
@@ -590,8 +749,72 @@ def _make_skill_archive_tool(user_id: str):
return _skill_archive
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
"""Run the curator consolidation pass as a React agent with skill tools bound."""
def _make_curator_skill_manage_tool():
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
"""
from langchain.tools import tool as lc_tool
from deerflow.tools.skill_manage_tool import _skill_manage_impl
@lc_tool("skill_manage")
def _curator_skill_manage(
action: str,
name: str,
content: str | None = None,
path: str | None = None,
find: str | None = None,
replace: str | None = None,
expected_count: int | None = None,
) -> str:
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
支持 action:create / patch / edit / write_file / remove_file / list / view。
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
Args:
action: 操作类型(见上;delete 会被拒绝)。
name: 连字符命名的技能名。
content: create / edit / write_file 的新文件内容。
path: write_file / remove_file 的支持文件路径。
find: patch 要替换的原文本。
replace: patch 的替换文本。
expected_count: patch 的预期替换次数(可选)。
"""
if action == "delete":
return (
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
)
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
return _run_async_blocking(
_skill_manage_impl(
runtime=None,
action=action,
name=name,
content=content,
path=path,
find=find,
replace=replace,
expected_count=expected_count,
)
)
return _curator_skill_manage
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
"""Run the curator consolidation pass as a React agent with skill tools bound.
返回 ``(llm_text, tool_actions)``:
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
"""
from deerflow.skills import curator_log
try:
@@ -601,10 +824,14 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.models import create_chat_model
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
model = create_chat_model()
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
curator_cfg = _get_curator_config(user_id)
# curator 评审用的模型:per-user 可配 curator.model_name(对应 config.yaml
# models[].name);为空则用主模型。
model_name = getattr(curator_cfg, "model_name", None) if curator_cfg else None
model = create_chat_model(name=model_name)
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
# (wrong action / missing required args), the exception is converted
# into a ToolMessage(status="error") so the LLM can read the error and
@@ -613,30 +840,58 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
timeout_seconds = getattr(curator_cfg, "review_timeout_seconds", 600) if curator_cfg else 600
curator_log.record(
"info",
f"LLM 评审:开始调用 React Agent(模型 {model_name or '主模型'},整体超时 {timeout_seconds}s)",
user_id=user_id,
)
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
# 给整个 React Agent 评审套一个总超时:agent.invoke 内部可能多轮调 LLM,模型层的
# 单次 request_timeout 兜不住整体。超时后本 curator 线程能正常收尾、释放运行锁,
# 而不是无限期挂起。注意:Python 无法强杀线程 —— 超时后那个 agent.invoke 线程会
# 成为孤儿(最终被模型层 request_timeout 或自然返回所终结),但不再拖住 curator。
# copy_context():把 current_user 等 ContextVar 传进工作线程,否则工具调用会把
# user_id 误解析为 DEFAULT_USER_ID。
ctx = contextvars.copy_context()
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="curator-llm")
future = pool.submit(ctx.run, agent.invoke, {"messages": [HumanMessage(content=prompt)]})
try:
result = future.result(timeout=timeout_seconds)
except concurrent.futures.TimeoutError:
curator_log.record(
"warning",
f"LLM 评审超时(超过 {timeout_seconds}s),已中止本次评审",
user_id=user_id,
)
logger.warning("Curator[%s] LLM review timed out after %ss", user_id, timeout_seconds)
return "", []
finally:
# wait=False:超时时不阻塞等待挂死的 invoke 线程。
pool.shutdown(wait=False)
messages = result.get("messages", [])
tool_actions = _extract_tool_actions(messages)
for msg in reversed(messages):
if not getattr(msg, "tool_calls", None):
content = getattr(msg, "content", "")
if isinstance(content, str) and content.strip():
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return content
return content, tool_actions
if isinstance(content, list):
text = " ".join(
block.get("text", "") if isinstance(block, dict) else str(block)
for block in content
).strip()
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return text
return text, tool_actions
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
return ""
return "", tool_actions
except Exception as e:
logger.warning("Curator LLM review failed: %s", e)
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
return ""
return "", []
def _parse_structured_summary(text: str) -> dict:
@@ -653,6 +908,44 @@ def _parse_structured_summary(text: str) -> dict:
return {"consolidations": [], "prunings": []}
def _extract_tool_actions(messages: list) -> list[dict]:
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
是否一致。每条形如 ``{tool, args, status, result}``。
"""
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
calls: dict[str, dict] = {}
for msg in messages:
for tc in getattr(msg, "tool_calls", None) or []:
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not tc_id:
continue
calls[tc_id] = {
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
}
# 再按 ToolMessage(带 tool_call_id)收集真实结果
actions: list[dict] = []
for msg in messages:
tc_id = getattr(msg, "tool_call_id", None)
if not tc_id:
continue
call = calls.get(tc_id, {})
raw = getattr(msg, "content", "")
result = raw if isinstance(raw, str) else str(raw)
actions.append(
{
"tool": call.get("tool") or getattr(msg, "name", None),
"args": call.get("args", {}),
"status": getattr(msg, "status", None) or "success",
"result": result[:500],
}
)
return actions
def run_curator_review(user_id: str) -> dict:
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
@@ -684,16 +977,23 @@ def run_curator_review(user_id: str) -> dict:
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
)
llm_text = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
if candidate_line_count == 0:
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
suggestions = {"consolidations": [], "prunings": []}
tool_actions: list[dict] = []
else:
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
# Surface per-item reasoning so operators can audit *why* each action
# was taken, not just the count.
for item in suggestions.get("consolidations", []) or []:
@@ -718,15 +1018,26 @@ def run_curator_review(user_id: str) -> dict:
user_id=user_id,
)
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
for act in tool_actions:
target = act.get("args", {}).get("name", "")
curator_log.record(
"info",
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
user_id=user_id,
)
state = _read_curator_state(user_id)
state["last_run_at"] = now.isoformat()
state["run_count"] = state.get("run_count", 0) + 1
state["last_run_summary"] = (
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
f"归档 {transition_counters['archived']} 个;"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
)
state["last_consolidation_suggestions"] = suggestions
state["last_consolidation_suggestions"] = suggestions # LLM 自述
state["last_actual_actions"] = tool_actions # 真实工具调用记录
_write_curator_state(state, user_id)
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
@@ -755,6 +1066,23 @@ def _curator_thread_target(user_id: str) -> None:
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.skills import curator_log
# 总开关 + curator 子开关:任一关闭则直接跳过。手动触发 /api/curator/run 也经由
# 本函数,因此手动触发同样受总开关与 curator.enabled 约束。
if not _skill_evolution_enabled(user_id):
logger.info("Curator[%s]: 技能进化总开关关闭,跳过", user_id)
return
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
logger.info("Curator[%s]: curator 未启用,跳过", user_id)
return
# 运行锁:防止同一用户的 curator 并发(定时器 vs 手动触发、连点手动触发)。抢不到
# 锁说明已有一轮在跑,直接跳过本次触发。
if not _try_acquire_run_lock(user_id, datetime.now(UTC)):
logger.info("Curator[%s]: 已有 curator 正在运行,跳过本次触发", user_id)
curator_log.record("info", "已有 curator 正在运行,跳过本次触发", user_id=user_id)
return
# Bind the per-thread user context for the entire curator run so every
# downstream tool / repository call resolves the correct user_id.
token = set_current_user(_CuratorUser(user_id))
@@ -765,13 +1093,16 @@ def _curator_thread_target(user_id: str) -> None:
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
finally:
reset_current_user(token)
# 无论成功或异常,必定释放运行锁。
_release_run_lock(user_id)
async def maybe_run_curator() -> None:
"""Async startup entry — kicks off a thread per user whose interval elapsed.
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
Skips users with no DB-tracked entries (a fresh deployment) and never
blocks lifespan startup on failure.
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
任何失败都不会阻断调用方。
"""
try:
from sqlalchemy import select
@@ -789,20 +1120,42 @@ async def maybe_run_curator() -> None:
logger.exception("Curator: failed to enumerate users at startup")
return
due_users: list[str] = []
for user_id in user_ids:
try:
if not should_run_now(user_id):
continue
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
daemon=True,
name=f"skill-curator-{user_id}",
)
thread.start()
if should_run_now(user_id):
due_users.append(user_id)
except Exception:
logger.exception("Curator: failed to spawn for user %s", user_id)
logger.exception("Curator: should_run_now check failed for user %s", user_id)
_dispatch_curator_runs(due_users)
def _dispatch_curator_runs(due_users: list[str]) -> None:
"""用线程池(并发上限 ``_CURATOR_MAX_CONCURRENCY``)派发到期用户的 curator 执行。
到期用户排队消费,而非一次性全部起线程齐调 LLM —— 防首次部署所有用户同时到期时
的惊群。``shutdown(wait=False)``:提交后立即返回,线程池在所有任务跑完后自行回收。
抽成独立函数便于对并发上限做单元测试。
"""
if not due_users:
return
logger.info(
"Curator: %d 个用户到期,以并发上限 %d 派发执行",
len(due_users),
_CURATOR_MAX_CONCURRENCY,
)
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=_CURATOR_MAX_CONCURRENCY,
thread_name_prefix="skill-curator",
)
try:
for user_id in due_users:
executor.submit(_curator_thread_target, user_id)
except Exception:
logger.exception("Curator: failed to submit curator tasks")
finally:
executor.shutdown(wait=False)
def get_curator_status(user_id: str) -> dict:
@@ -819,7 +1172,7 @@ def get_curator_status(user_id: str) -> dict:
s = entry.get("state", STATE_ACTIVE)
counts[s] = counts.get(s, 0) + 1
cfg = _get_curator_config()
cfg = _get_curator_config(user_id)
return {
"enabled": cfg is not None and getattr(cfg, "enabled", True),
"run_count": state.get("run_count", 0),
@@ -832,4 +1185,8 @@ def get_curator_status(user_id: str) -> dict:
"stale": counts[STATE_STALE],
"archived": counts[STATE_ARCHIVED],
},
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
or {"consolidations": [], "prunings": []},
"last_actual_actions": state.get("last_actual_actions") or [],
}
@@ -50,8 +50,10 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
try:
config = app_config or get_app_config()
model_name = config.skill_evolution.moderation_model_name
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config) if model_name else create_chat_model(thinking_enabled=False, app_config=config)
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
model_name = config.skill_evolution.curator.model_name
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config)
response = await model.ainvoke(
[
{"role": "system", "content": rubric},
@@ -6,6 +6,7 @@ import json
import logging
import os
import tempfile
import threading
from datetime import UTC, datetime
from pathlib import Path
@@ -13,6 +14,11 @@ logger = logging.getLogger(__name__)
_USAGE_FILENAME = ".usage.json"
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
# 单进程跑,故进程内锁即可保证状态流转不丢。
_usage_lock = threading.Lock()
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
@@ -75,13 +81,20 @@ def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _update_entry(name: str, updates: dict) -> None:
def _update_entry(name: str, updates) -> None:
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
避免并发下的 lost update。
"""
try:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
entry.update(updates)
data[name] = entry
_write_all(data)
with _usage_lock:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
resolved = updates(entry) if callable(updates) else updates
entry.update(resolved)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
@@ -103,18 +116,16 @@ def agent_created_entries() -> dict:
def bump_view(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
def bump_use(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
def bump_patch(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
def set_state(name: str, state: str) -> None:
@@ -132,9 +143,10 @@ def set_pinned(name: str, pinned: bool) -> None:
def forget(name: str) -> None:
try:
data = _read_all()
data.pop(name, None)
_write_all(data)
with _usage_lock:
data = _read_all()
data.pop(name, None)
_write_all(data)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
@@ -59,6 +59,66 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
return None
def _resolve_effective_user_id() -> str | None:
"""当前请求/线程绑定的用户 id;解析失败时返回 None。"""
try:
from deerflow.runtime.user_context import get_effective_user_id
return get_effective_user_id()
except Exception:
logger.debug("Failed to resolve effective user id for skill_list", exc_info=True)
return None
def _run_coro_blocking(coro):
"""在同步工具里把一个协程跑到完成。
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
执行 ``asyncio.run``,否则直接跑。
"""
import asyncio
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
用于默认 lead agent 的可见性过滤:只算"用户自己拥有的"自定义技能,不含
其他用户发布的。返回 ``None`` 表示 ``SkillStore`` 不可用(未配置数据库
等)—— 调用方应降级为"放行全部自定义技能",避免离线无库部署下技能凭空
消失。
"""
if not user_id:
return None
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = _run_coro_blocking(store.list_visible(user_id))
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
return None
@tool("skill_view")
def skill_view_tool(name: str) -> str:
"""Read the full SKILL.md content of a skill by name.
@@ -106,28 +166,47 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
under the ``skills:`` field — the agent only sees what it was wired to
use.
For the **default lead agent** (no agent_id in runtime), there is no
per-agent whitelist and every visible skill is returned.
For the **default lead agent** (no agent_id in runtime), the result is
filtered to non-archived skills that are either built-in (public) or
owned by the current user. Other users' published custom skills and
archived skills are excluded.
Returns a JSON array with fields: name, description, category, enabled,
state, use_count, view_count, patch_count, last_activity_at, pinned,
source.
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import all_entries, latest_activity_at
from deerflow.skills.types import SkillCategory
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
agent_id = _resolve_active_agent_id(runtime)
allowed = _load_agent_skill_whitelist(agent_id)
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
# 当前用户"的技能。owned 为 None 表示 SkillStore 不可用,降级放行全部自定义
# 技能(见 _load_owned_skill_names)。
owned: set[str] | None = None
if allowed is None:
owned = _load_owned_skill_names(_resolve_effective_user_id())
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=False)
usage = all_entries()
result = []
for skill in skills:
if allowed is not None and skill.name not in allowed:
continue
entry = usage.get(skill.name, {})
if allowed is not None:
# 自定义 agent:严格按 config.yaml 的 skills 白名单过滤。
if skill.name not in allowed:
continue
else:
# 默认 agent:跳过已归档技能。
if entry.get("state") == STATE_ARCHIVED:
continue
# 自定义技能仅当属于当前用户时可见;内置 public 技能始终可见。
if skill.category == SkillCategory.CUSTOM and owned is not None and skill.name not in owned:
continue
result.append({
"name": skill.name,
"description": skill.description,
@@ -103,11 +103,26 @@ def get_available_tools(
builtin_tools.extend(HINDSIGHT_TOOLS)
logger.info("Including Hindsight tools (memory_mode=%s)", memory_mode)
skill_evolution_config = getattr(config, "skill_evolution", None)
# skill_list / skill_view 是只读工具(发现、查看技能),与"自我进化"写能力无关,
# 始终提供给 agent。
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
builtin_tools.extend([skill_list_tool, skill_view_tool])
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
from deerflow.runtime.user_context import get_effective_user_id
skill_evolution_config = load_skill_evolution_config_for_user(get_effective_user_id())
except Exception: # noqa: BLE001 - 解析失败不能阻断 agent 构建
skill_evolution_config = getattr(config, "skill_evolution", None)
if getattr(skill_evolution_config, "enabled", False):
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
builtin_tools.append(skill_manage_tool)
# Add subagent tools only if enabled via runtime parameter
if subagent_enabled: