Merge remote-tracking branch 'origin/master' into ji

This commit is contained in:
吉鑫升 committed 2026-05-18 10:03:49 +08:00
commit 4761677676
74 files changed
+9770 -2887

No files matched your search

@@ -302,6 +302,55 @@ Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.age
- **Injection**: Enabled skills listed in agent system prompt with container paths
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
#### Skill Curator (`packages/harness/deerflow/skills/curator.py`)
Background skill lifecycle manager. The Gateway `lifespan` starts a resident
scheduler task (`_curator_scheduler_loop` in `app/gateway/app.py`) that — after a
5-minute startup delay — calls `maybe_run_curator()` every hour. A cross-process
file lock (`.curator_scheduler.lock`) elects a single leader worker so the
scheduler runs once per deployment. Per run:
- **Master switch** — `skill_evolution.enabled` (default **`false`**) gates the
whole feature; `curator.enabled` (default **`false`**) gates the curator
specifically. Both must be `true` for the curator to act. `should_run_now()`
and `_curator_thread_target()` both enforce this — so manual `POST
/api/curator/run` also respects the master switch. Config is
`config.yaml.skill_evolution` deep-merged with per-user
`users/{user_id}/skill_evolution.json`.
- **Per-user run lock** — `.curator_state` carries `running` / `started_at`;
`_try_acquire_run_lock()` prevents the same user's curator from running
concurrently (scheduler vs. manual trigger, or double manual clicks). A lock
whose `started_at` is older than `_CURATOR_STALE_LOCK_HOURS` (6h) is treated as
process-crash residue and can be taken over.
- **Concurrency cap** — `maybe_run_curator()` collects all due users and hands
them to `_dispatch_curator_runs()`, which runs them through a
`ThreadPoolExecutor` capped at `_CURATOR_MAX_CONCURRENCY` (5) — avoids a
thundering herd of LLM calls when many users come due at once (e.g. first
deploy).
- **Failure back-off** — `_release_run_lock()` always stamps `last_attempt_at`.
When a first run keeps crashing (`last_run_at` never set), `should_run_now()`
retries on `_CURATOR_RETRY_HOURS` (6h) instead of every hourly tick.
- **`should_run_now()`** is the real gate: the hourly tick only checks "is the
per-user `interval_hours` (default 168h) elapsed". Timestamp parsing tolerates
naive (tz-less) strings — a corrupt `last_run_at` triggers a re-run rather than
a permanent deadlock.
- **Published-skill filter** — `SkillStore.list_published_names()` is a batch
query; the curator fetches the published-name set once per run instead of one
DB round-trip per candidate skill.
- **LLM review timeout** — `_run_llm_review()` runs `agent.invoke()` in a worker
thread bounded by `curator.review_timeout_seconds` (default 600s). The model's
per-request `request_timeout` can't bound a multi-turn React agent loop, so on
timeout the run is abandoned (a warning is written to `curator_log`, visible in
the Web UI) and the curator finishes cleanly + releases the run lock. The
worker context is copied via `contextvars.copy_context()` so tool calls still
resolve the correct `user_id`.
- **Review model** — `_run_llm_review()` uses `curator.model_name` (a
`config.yaml` `models[]` entry name) if set, else the primary model; it is
editable from the Web UI evolution panel. The skill security scanner
(`scan_skill_content` in `security_scanner.py`) reuses the same
`skill_evolution.curator.model_name` setting — review and moderation share one
model. There is no separate moderation-model config field.
### Model Factory (`packages/harness/deerflow/models/factory.py`)
- `create_chat_model(name, thinking_enabled)` instantiates LLM from config via reflection
@@ -1,6 +1,7 @@
import asyncio
import logging
import os
import sys
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
@@ -268,6 +269,93 @@ async def _run_db_migrations() -> None:
logger.exception("Database migration failed — schema may be out of date")
# Skill curator 调度参数。
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
def _acquire_singleton_lock(lock_path):
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
进程异常退出时 OS 会自动释放该锁。
"""
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = open(lock_path, "a+")
except OSError:
logger.exception("Curator scheduler: 打开锁文件失败")
return None
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
return None
return handle
def _release_singleton_lock(handle) -> None:
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
try:
if sys.platform == "win32":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError:
pass
finally:
try:
handle.close()
except OSError:
pass
async def _curator_scheduler_loop() -> None:
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
"""
from deerflow.config.runtime_paths import runtime_home
from deerflow.skills.curator import maybe_run_curator
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
if lock_handle is None:
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
return
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
try:
# 启动后延迟一会儿再首次检查,避开启动高峰
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
while True:
try:
await maybe_run_curator()
except Exception:
logger.exception("Curator scheduler tick failed (non-fatal)")
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
finally:
_release_singleton_lock(lock_handle)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
@@ -302,12 +390,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill ownership migration failed (non-fatal)")
# Kick off skill curator in background if the run interval has elapsed
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
try:
from deerflow.skills.curator import maybe_run_curator
asyncio.create_task(maybe_run_curator())
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
except Exception:
logger.exception("Skill curator startup task failed (non-fatal)")
logger.exception("Skill curator scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
@@ -335,6 +423,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
yield
# 停止 curator 调度器
curator_task = getattr(app.state, "curator_scheduler_task", None)
if curator_task is not None:
curator_task.cancel()
try:
await curator_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Curator scheduler shutdown error")
try:
scheduler_service = getattr(app.state, "scheduled_task_service", None)
if scheduler_service is not None:
@@ -3,9 +3,50 @@ from pydantic import BaseModel, Field
from app.gateway.deps import get_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import ModelConfig
router = APIRouter(prefix="/api", tags=["models"])
_PROVIDER_KEYWORDS: list[tuple[str, str]] = [
("langchain_openai", "openai"),
("langchain_anthropic", "anthropic"),
("langchain_google", "google"),
("langchain_groq", "groq"),
("langchain_mistralai", "mistral"),
("langchain_cohere", "cohere"),
("ollama", "ollama"),
("vllm", "vllm"),
]
_LOCAL_PROVIDERS = {"ollama", "vllm"}
def _derive_provider(use: str) -> str | None:
use_lower = use.lower()
for keyword, provider in _PROVIDER_KEYWORDS:
if keyword in use_lower:
return provider
return None
def _build_model_response(model: ModelConfig) -> "ModelResponse":
extra = model.model_extra or {}
provider = str(extra["provider"]) if "provider" in extra else _derive_provider(model.use)
is_local = bool(extra["is_local"]) if "is_local" in extra else (provider in _LOCAL_PROVIDERS)
supports_tool_calling = bool(extra["supports_tool_calling"]) if "supports_tool_calling" in extra else True
return ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
supports_vision=model.supports_vision,
supports_tool_calling=supports_tool_calling,
is_local=is_local,
provider=provider,
)
class ModelResponse(BaseModel):
"""Response model for model information."""
@@ -16,6 +57,10 @@ class ModelResponse(BaseModel):
description: str | None = Field(None, description="Model description")
supports_thinking: bool = Field(default=False, description="Whether model supports thinking mode")
supports_reasoning_effort: bool = Field(default=False, description="Whether model supports reasoning effort")
supports_vision: bool = Field(default=False, description="Whether model supports image inputs")
supports_tool_calling: bool = Field(default=True, description="Whether model supports tool/function calling")
is_local: bool = Field(default=False, description="Whether this is a locally-hosted model")
provider: str | None = Field(None, description="Model provider name (openai, anthropic, ollama, etc.)")
class TokenUsageResponse(BaseModel):
@@ -73,17 +118,7 @@ async def list_models(config: AppConfig = Depends(get_config)) -> ModelsListResp
}
```
"""
models = [
ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
)
for model in config.models
]
models = [_build_model_response(model) for model in config.models]
return ModelsListResponse(
models=models,
token_usage=TokenUsageResponse(enabled=config.token_usage.enabled),
@@ -122,11 +157,4 @@ async def get_model(model_name: str, config: AppConfig = Depends(get_config)) ->
if model is None:
raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
return ModelResponse(
name=model.name,
model=model.model,
display_name=model.display_name,
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=model.supports_reasoning_effort,
)
return _build_model_response(model)
@@ -1045,12 +1045,21 @@ async def curator_status(request: Request) -> dict:
@router.post("/curator/run", summary="Trigger Curator Run")
async def run_curator(request: Request) -> dict:
"""Manually trigger a curator review pass for the authenticated user
(bypasses interval check, runs in a background thread)."""
(bypasses the interval check, runs in a background thread).
Pre-checks the master switch / curator switch / run lock so the response
accurately reflects whether a run actually started — instead of always
reporting success. Returns ``{"triggered": bool, "reason": str | None}``.
"""
try:
import threading
from deerflow.skills.curator import _curator_thread_target
from deerflow.skills.curator import _curator_thread_target, manual_trigger_blocked_reason
user_id = _current_user_id(request)
reason = manual_trigger_blocked_reason(user_id)
if reason is not None:
return {"triggered": False, "reason": reason}
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
@@ -1058,7 +1067,7 @@ async def run_curator(request: Request) -> dict:
name=f"skill-curator-manual-{user_id}",
)
thread.start()
return {"triggered": True, "message": "Curator run started in background"}
return {"triggered": True, "reason": None}
except Exception as e:
logger.error("Failed to trigger curator: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
@@ -1067,17 +1076,19 @@ async def run_curator(request: Request) -> dict:
class CuratorSubConfig(BaseModel):
"""Nested curator configuration mirroring CuratorConfig."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
# 与 CuratorConfig.enabled 保持一致:默认关闭,需显式 opt-in。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
review_timeout_seconds: int = Field(default=600, ge=1, description="Overall timeout (seconds) for the LLM review pass.")
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass; null = primary model.")
class CuratorConfigResponse(BaseModel):
"""Response model exposing the effective skill_evolution config."""
enabled: bool
moderation_model_name: str | None
curator: CuratorSubConfig
@@ -1089,7 +1100,6 @@ class CuratorConfigRequest(BaseModel):
"""
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
curator: CuratorSubConfig
@@ -1113,12 +1123,13 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
"""
return CuratorConfigResponse(
enabled=evo.enabled,
moderation_model_name=evo.moderation_model_name,
curator=CuratorSubConfig(
enabled=evo.curator.enabled,
interval_hours=evo.curator.interval_hours,
stale_after_days=evo.curator.stale_after_days,
archive_after_days=evo.curator.archive_after_days,
review_timeout_seconds=evo.curator.review_timeout_seconds,
model_name=evo.curator.model_name,
),
)
@@ -98,6 +98,10 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]:
_DEFAULT_ASSISTANT_ID = "lead_agent"
# Built-in LangGraph graph IDs registered in langgraph.json.
# These are NOT stored in the agents DB and must bypass the custom-agent access check.
_BUILTIN_GRAPH_IDS: frozenset[str] = frozenset({"lead_agent", "canvas_agent"})
def _prepend_prompt_prefix(graph_input: dict[str, Any], prefix: str) -> dict[str, Any]:
"""Return ``graph_input`` with the prefix *tagged* onto its first HumanMessage.
@@ -180,12 +184,15 @@ def merge_run_context_overrides(config: dict[str, Any], context: Mapping[str, An
def resolve_agent_factory(assistant_id: str | None):
"""Resolve the agent factory callable from config.
Custom agents are implemented as ``lead_agent`` + an ``agent_id``
injected into ``configurable`` or ``context`` — see
:func:`build_run_config`. All ``assistant_id`` values therefore map to the
same factory; the routing happens inside ``make_lead_agent`` when it reads
``cfg["agent_id"]``.
Built-in graphs (``lead_agent``, ``canvas_agent``) are dispatched directly
to their own factory. Custom agents share the ``lead_agent`` factory with
``agent_id`` injected into configurable/context by :func:`build_run_config`.
"""
if assistant_id == "canvas_agent":
from deerflow.agents.canvas_agent.agent import make_canvas_agent
return make_canvas_agent
from deerflow.agents.lead_agent.agent import make_lead_agent
return make_lead_agent
@@ -244,8 +251,9 @@ def build_run_config(
config["configurable"] = {"thread_id": thread_id}
# Inject custom agent id when the caller specified a non-default assistant.
# Honour an explicit agent_id in the active runtime options container.
if assistant_id and assistant_id != _DEFAULT_ASSISTANT_ID:
# Built-in graphs (lead_agent, canvas_agent) are dispatched via their own
# factory and must NOT receive an agent_id override.
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
normalized = assistant_id.strip()
if not normalized or not re.fullmatch(r"[A-Za-z0-9_-]+", normalized):
raise ValueError(f"Invalid assistant_id {assistant_id!r}: must contain only letters, digits, hyphens, and underscores.")
@@ -265,7 +273,7 @@ def build_run_config(
def _collect_requested_agent_ids(body: Any) -> set[str]:
"""Collect agent ids from assistant_id/config/context for access checks."""
agent_ids: set[str] = set()
if body.assistant_id and body.assistant_id != _DEFAULT_ASSISTANT_ID:
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
agent_ids.add(str(body.assistant_id))
def collect(container: Mapping[str, Any] | None) -> None:
@@ -6,7 +6,8 @@
],
"env": ".env",
"graphs": {
"lead_agent": "deerflow.agents:make_lead_agent"
"lead_agent": "deerflow.agents:make_lead_agent",
"canvas_agent": "deerflow.agents.canvas_agent:make_canvas_agent"
},
"auth": {
"path": "./app/gateway/langgraph_auth.py:auth"
@@ -0,0 +1,3 @@
from deerflow.agents.canvas_agent.agent import make_canvas_agent
__all__ = ["make_canvas_agent"]
@@ -0,0 +1,68 @@
from __future__ import annotations
import logging
from langchain.agents import create_agent
from langchain_core.runnables import RunnableConfig
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
from deerflow.agents.canvas_agent.tools import update_artifact_tool
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.agents.thread_state import ThreadState
from deerflow.config.app_config import get_app_config
from deerflow.models import create_chat_model
from deerflow.tools.builtins import ask_clarification_tool
logger = logging.getLogger(__name__)
def _get_runtime_config(config: RunnableConfig) -> dict:
cfg = dict(config.get("configurable", {}) or {})
context = config.get("context", {}) or {}
if isinstance(context, dict):
cfg.update(context)
return cfg
def make_canvas_agent(config: RunnableConfig):
"""LangGraph graph factory for canvas mode; signature compatible with LangGraph Server."""
app_config = get_app_config()
cfg = _get_runtime_config(config)
requested_model = cfg.get("model_name") or cfg.get("model")
if requested_model and app_config.get_model_config(requested_model):
model_name = requested_model
else:
if requested_model:
logger.warning("Canvas agent: model '%s' not found; falling back to default.", requested_model)
model_name = app_config.models[0].name if app_config.models else None
if model_name is None:
raise ValueError("No chat models configured. Please add at least one model in config.yaml.")
thinking_enabled = cfg.get("thinking_enabled", False)
model_config = app_config.get_model_config(model_name)
if thinking_enabled and model_config and not model_config.supports_thinking:
thinking_enabled = False
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
middlewares = [
DanglingToolCallMiddleware(),
ToolErrorHandlingMiddleware(),
TitleMiddleware(app_config=app_config),
LoopDetectionMiddleware(),
ClarificationMiddleware(),
]
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
tools=[update_artifact_tool, ask_clarification_tool],
middleware=middlewares,
system_prompt=CANVAS_SYSTEM_PROMPT,
state_schema=ThreadState,
)
@@ -0,0 +1,40 @@
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
<canvas_behavior>
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
ALWAYS call `update_artifact` to save your work. Never embed the artifact content in your chat response.
Rules:
- On every request that creates or changes the artifact, call `update_artifact` ONCE with the COMPLETE updated content.
- In your text reply, describe what you did in 1–2 sentences.
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
</canvas_behavior>
<artifact_context>
The current artifact state is injected into your context. When editing, build on the existing content exactly.
When creating from scratch, produce a complete, high-quality artifact in one shot.
</artifact_context>
<selection_rewrite>
When the user's message contains a <selection> block, you MUST:
1. Identify the exact selection in the current artifact content.
2. Rewrite ONLY that selection according to the user's instruction.
3. Keep the rest of the artifact exactly unchanged.
4. Call update_artifact with the complete artifact where the selection has been replaced.
Example: if the artifact is "A B C" and the selection is "B" with instruction "make it bold",
produce "A **B** C" — not a new document.
</selection_rewrite>
<quick_actions>
When the user triggers a quick action, apply it to the full artifact unless specific highlighted text is provided:
- expand: Add depth and detail while preserving structure
- shorten: Condense to the essential points
- formalize: Rewrite in professional, formal language
- casualize: Rewrite in friendly, conversational language
- add_comments: Add clear, helpful code comments
- add_logs: Add appropriate logging statements
- fix_bugs: Identify and fix all bugs in the code
- optimize_code: Improve performance and readability
</quick_actions>
"""
@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import Annotated, Literal
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
from langchain_core.messages import ToolMessage
from langgraph.types import Command
from langgraph.typing import ContextT
from deerflow.agents.thread_state import CanvasArtifact, CanvasArtifactCode, CanvasArtifactText, ThreadState
@tool("update_artifact", parse_docstring=True)
def update_artifact_tool(
runtime: ToolRuntime[ContextT, ThreadState],
tool_call_id: Annotated[str, InjectedToolCallId],
type: Literal["text", "code"],
title: str,
content: str,
language: str = "other",
) -> Command:
"""Create or update the canvas artifact. Always call this to persist your work.
Args:
type: Artifact type — "text" for markdown documents, "code" for source code.
title: Human-readable title of the artifact.
content: Full artifact content: markdown text or complete source code.
language: Programming language (for code type only, e.g. "python", "typescript"). Ignored for text.
"""
current: CanvasArtifact | None = (runtime.state or {}).get("artifact") if runtime.state else None
if current is not None:
existing_contents = list(current.get("contents", []))
new_index = current.get("currentIndex", len(existing_contents)) + 1
else:
existing_contents = []
new_index = 1
if type == "text":
new_entry: CanvasArtifactText | CanvasArtifactCode = CanvasArtifactText(
index=new_index,
type="text",
title=title,
fullMarkdown=content,
)
else:
new_entry = CanvasArtifactCode(
index=new_index,
type="code",
title=title,
language=language,
code=content,
)
updated: CanvasArtifact = {
"currentIndex": new_index,
"contents": existing_contents + [new_entry],
}
return Command(
update={
"artifact": updated,
"messages": [ToolMessage("Artifact updated.", tool_call_id=tool_call_id)],
}
)
@@ -69,8 +69,25 @@ _enabled_skills_refresh_version = 0
_enabled_skills_refresh_event = threading.Event()
def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
"""从注入提示词的技能列表里剔除已归档(usage state=archived)的技能。
技能归档是软状态:SKILL.md 仍留在 custom/<name>/、enabled 标志不变,所以
load_skills(enabled_only=True) 仍会返回它。但归档技能不应再出现在 agent 的系统
提示词里 —— 对 agent 而言它应等同于"不存在"。失败时不过滤(返回原列表)。
"""
try:
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
entries = all_entries()
return [s for s in skills if entries.get(s.name, {}).get("state") != STATE_ARCHIVED]
except Exception:
logger.debug("Failed to filter archived skills for prompt injection", exc_info=True)
return skills
def _load_enabled_skills_sync() -> list[Skill]:
return list(get_or_new_skill_storage().load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage().load_skills(enabled_only=True)))
def _start_enabled_skills_refresh_thread() -> None:
@@ -171,7 +188,7 @@ def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[
"""
if app_config is None:
return _get_enabled_skills()
return list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True)))
def _skill_mutability_label(category: SkillCategory | str) -> str:
@@ -479,11 +496,11 @@ You: "Deploying to staging..." [proceed]
{skills_section}
{deferred_tools_section}
{writing_mode_section}
{subagent_section}
{deferred_tools_section}
{writing_mode_section}
{subagent_section}
<working_directory existed="true">
- User uploads: `/mnt/user-data/uploads` - Files uploaded by the user (automatically listed in context)
@@ -796,17 +813,17 @@ def _build_custom_mounts_section(*, app_config: AppConfig | None = None) -> str:
return f"\n**Custom Mounted Directories:**\n{mounts_list}\n- If the user needs files outside `/mnt/user-data`, use these absolute container paths directly when they match the requested directory"
def apply_prompt_template(
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
*,
agent_id: str | None = None,
agent_name: str | None = None,
writing_mode: bool = False,
writing_artifact_path: str | None = None,
available_skills: set[str] | None = None,
app_config: AppConfig | None = None,
) -> str:
def apply_prompt_template(
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
*,
agent_id: str | None = None,
agent_name: str | None = None,
writing_mode: bool = False,
writing_artifact_path: str | None = None,
available_skills: set[str] | None = None,
app_config: AppConfig | None = None,
) -> str:
storage_agent_id = agent_id if agent_id is not None else agent_name
# Get memory context
@@ -838,33 +855,33 @@ def apply_prompt_template(
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
current_writing_artifact = (
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
if writing_artifact_path
else ""
)
writing_mode_section = (
f"""
<writing_mode>
Writing mode is enabled for this run.
**Mandatory Output Contract**
- Treat the user's request as a writing/report-generation task.
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
{current_writing_artifact}
</writing_mode>
"""
if writing_mode
else ""
)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
current_writing_artifact = (
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
if writing_artifact_path
else ""
)
writing_mode_section = (
f"""
<writing_mode>
Writing mode is enabled for this run.
**Mandatory Output Contract**
- Treat the user's request as a writing/report-generation task.
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
{current_writing_artifact}
</writing_mode>
"""
if writing_mode
else ""
)
# Build ACP agent section only if ACP agents are configured
acp_section = _build_acp_section(app_config=app_config)
@@ -878,10 +895,10 @@ Writing mode is enabled for this run.
prompt = SYSTEM_PROMPT_TEMPLATE.format(
agent_name=agent_name or storage_agent_id or "CM 助手",
soul=get_agent_soul(storage_agent_id),
skills_section=skills_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
skills_section=skills_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
network_policy_section=network_policy_section,
subagent_section=subagent_section,
subagent_reminder=subagent_reminder,
@@ -1,4 +1,4 @@
from typing import Annotated, NotRequired, TypedDict
from typing import Annotated, Literal, NotRequired, Optional, TypedDict
from langchain.agents import AgentState
@@ -18,6 +18,26 @@ class ViewedImageData(TypedDict):
mime_type: str
class CanvasArtifactText(TypedDict):
index: int
type: Literal["text"]
title: str
fullMarkdown: str
class CanvasArtifactCode(TypedDict):
index: int
type: Literal["code"]
title: str
language: str
code: str
class CanvasArtifact(TypedDict):
currentIndex: int
contents: list[CanvasArtifactText | CanvasArtifactCode]
def merge_artifacts(existing: list[str] | None, new: list[str] | None) -> list[str]:
"""Reducer for artifacts list - merges and deduplicates artifacts."""
if existing is None:
@@ -53,3 +73,11 @@ class ThreadState(AgentState):
todos: NotRequired[list | None]
uploaded_files: NotRequired[list[dict] | None]
viewed_images: Annotated[dict[str, ViewedImageData], merge_viewed_images] # image_path -> {base64, mime_type}
# Canvas fields (coexist with artifacts: list[str] which tracks file outputs)
artifact: NotRequired[Optional[CanvasArtifact]]
highlighted_text: NotRequired[Optional[str]]
highlighted_code: NotRequired[Optional[str]]
canvas_mode: NotRequired[Optional[Literal["text", "code"]]]
quick_action_id: NotRequired[Optional[str]]
web_search_enabled: NotRequired[bool]
web_search_results: NotRequired[Optional[list[dict]]]
@@ -4,10 +4,17 @@ from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
@@ -17,10 +24,6 @@ class SkillEvolutionConfig(BaseModel):
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
moderation_model_name: str | None = Field(
default=None,
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
@@ -67,3 +67,13 @@ class SkillStore(ABC):
async def count_owned(self, user_id: str) -> int:
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def list_published_names(self) -> set[str]:
"""Return the set of skill names with ``published = True`` (any owner).
Batch alternative to calling ``get_any`` per skill — the curator uses
this to filter out published skills in a single query instead of one
DB round-trip per candidate skill.
"""
raise NotImplementedError
@@ -133,3 +133,6 @@ class MemorySkillStore(SkillStore):
async def count_owned(self, user_id: str) -> int:
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
async def list_published_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("published")}
@@ -148,3 +148,9 @@ class SkillRepository(SkillStore):
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def list_published_names(self) -> set[str]:
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
async with self._sf() as session:
result = await session.execute(stmt)
return {row[0] for row in result.all()}
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import concurrent.futures
import contextvars
import json
import logging
import os
@@ -25,6 +26,10 @@ logger = logging.getLogger(__name__)
_CURATOR_STATE_FILENAME = ".curator_state"
# 周期调度一轮里同时执行的 curator 数量上限。到期用户多时(尤其首次部署所有用户一起
# 到期)用线程池排队消费,避免一次性起 N 个线程齐调 LLM 打爆 provider 速率限制。
_CURATOR_MAX_CONCURRENCY = 5
_T = TypeVar("_T")
@@ -119,6 +124,81 @@ def _write_curator_state(state: dict, user_id: str | None) -> None:
logger.debug("Failed to write curator state", exc_info=True)
def _parse_state_time(value: str | None) -> datetime | None:
"""解析 ``.curator_state`` 里的 ISO 时间字符串,返回 aware datetime(UTC)。
naive(无时区)字符串按 UTC 补全 —— 与 ``apply_automatic_transitions`` 的处理
一致,避免 naive 时间串与 ``datetime.now(UTC)`` 直接比较时抛异常。无法解析时
返回 None,由调用方决定降级行为。
"""
if not value:
return None
try:
dt = datetime.fromisoformat(value)
except Exception:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=UTC)
return dt
# Curator 运行锁:防止同一用户的 curator 并发执行(定时器 vs 手动触发、连点手动触发)。
# 锁状态写进 .curator_state 的 running/started_at 字段,跨 worker 可见(手动请求可能
# 落到非调度 worker)。started_at 超过下面的阈值视为进程残留,允许接管以免永久死锁。
_CURATOR_STALE_LOCK_HOURS = 6
# 上一轮"尝试"过但未成功(last_run_at 始终为空)时的重试间隔 —— 比正常 interval_hours
# 短得多,但也不至于每个调度 tick(1h)都重试、每次都烧一份 LLM 配额。
_CURATOR_RETRY_HOURS = 6
# 进程内串行化运行锁的"读-改-写",保证同进程内的抢锁是原子的(跨进程的极小竞态窗口
# 可接受 —— curator 触发频率极低)。
_run_lock_guard = threading.Lock()
def _run_lock_active(state: dict, now: datetime) -> bool:
"""判断 ``.curator_state`` 里的运行锁当前是否有效(running 且未过期)。
``started_at`` 缺失或不可解析、或距今已超过 ``_CURATOR_STALE_LOCK_HOURS`` 时,
视为进程残留留下的死锁,返回 False 以便后续触发能接管。
"""
if not state.get("running"):
return False
started_dt = _parse_state_time(state.get("started_at"))
if started_dt is None:
return False
return now < started_dt + timedelta(hours=_CURATOR_STALE_LOCK_HOURS)
def _try_acquire_run_lock(user_id: str, now: datetime) -> bool:
"""尝试为 ``user_id`` 抢占 curator 运行锁。
抢到锁(已把 running=True / started_at 写回)返回 True;已有未过期的运行时返回
False。读-改-写在 ``_run_lock_guard`` 下进行,保证同进程内原子。
"""
with _run_lock_guard:
state = _read_curator_state(user_id)
if _run_lock_active(state, now):
return False
state["running"] = True
state["started_at"] = now.isoformat()
_write_curator_state(state, user_id)
return True
def _release_run_lock(user_id: str) -> None:
"""释放 curator 运行锁并记录本次尝试时间。无论运行成功或异常都应调用。
除了把 running 置回 False,还写入 ``last_attempt_at`` —— 用于失败退避:首次运行
若一直崩溃,``last_run_at`` 永远为空,``should_run_now`` 改用
``last_attempt_at + _CURATOR_RETRY_HOURS`` 判定,避免每个 tick 都重试。
成功跑完后 ``last_run_at`` 已被写入,``should_run_now`` 不再看 ``last_attempt_at``。
"""
with _run_lock_guard:
state = _read_curator_state(user_id)
state["running"] = False
state["last_attempt_at"] = datetime.now(UTC).isoformat()
_write_curator_state(state, user_id)
def _get_curator_config(user_id: str | None = None):
"""Return the curator config object for ``user_id``.
@@ -135,6 +215,21 @@ def _get_curator_config(user_id: str | None = None):
return None
def _skill_evolution_enabled(user_id: str | None = None) -> bool:
"""返回技能进化总开关 ``skill_evolution.enabled``(已合并 per-user 覆盖)。
这是整套技能进化功能的总闸:关闭时 curator 完全不动作。任何读取异常都按
"关闭"处理(fail-safe)—— 宁可不跑,也不在配置不可读时偷偷运行。
"""
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
skill_evo = load_skill_evolution_config_for_user(user_id)
return bool(getattr(skill_evo, "enabled", False))
except Exception:
return False
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
"""Return skill names owned by ``user_id`` (used to filter transitions).
@@ -231,12 +326,39 @@ def _is_skill_published(skill_name: str) -> bool:
return False
def _list_published_skill_names() -> set[str]:
"""一次性返回所有 ``published=True`` 的技能名集合。
取代"每个技能调一次 ``_is_skill_published``"的逐条查库 —— curator 一轮里会遍历
所有候选技能,逐条查会产生 ≈2N 次独立 event loop + DB 往返。出错时返回空集
(fail-open:与 ``_is_skill_published`` 出错返回 False 语义一致,即 DB 不可用时
所有技能都视为未发布,回退到 curator 的 legacy 行为)。
"""
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return set()
store = make_skill_store(factory)
async def _run() -> set[str]:
return await store.list_published_names()
return _run_async_blocking(_run())
except Exception:
logger.debug("_list_published_skill_names failed", exc_info=True)
return set()
def _eligible_for_curation(
name: str,
entry: dict,
*,
owned_set: set[str] | None,
referenced_set: set[str],
published_set: set[str],
) -> bool:
"""Three-condition gate for whether curator may touch a skill.
@@ -244,11 +366,15 @@ def _eligible_for_curation(
1. source == "agent" (caller is expected to have already filtered, but
we double-check here for safety)
2. NOT referenced by any agent's skills config
3. NOT published (queried via SkillStore)
3. NOT published
``owned_set`` is the set of skill names owned by the running user. When
None, the legacy "process every agent-created skill" fallback applies
for ownership only (the other two conditions still hold).
``published_set`` is the set of published skill names, pre-fetched once by
the caller via ``_list_published_skill_names()`` — avoids a per-skill DB
round-trip.
"""
if entry.get("source") != "agent":
return False
@@ -256,7 +382,7 @@ def _eligible_for_curation(
return False
if name in referenced_set:
return False
if _is_skill_published(name):
if name in published_set:
return False
return True
@@ -264,21 +390,52 @@ def _eligible_for_curation(
def should_run_now(user_id: str) -> bool:
"""Return True if the curator interval has elapsed for ``user_id`` and it
is not paused."""
curator_cfg = _get_curator_config()
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
# 总开关:技能进化整体关闭时,curator 一律不跑。
if not _skill_evolution_enabled(user_id):
return False
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
return False
state = _read_curator_state(user_id)
if state.get("paused"):
return False
# 运行锁:已有 curator 在跑(且锁未过期)时不再触发新的一轮。
if _run_lock_active(state, datetime.now(UTC)):
return False
last_run = state.get("last_run_at")
if not last_run:
return True
# 从未成功跑完过。若此前已尝试过(last_attempt_at 存在),按较短的重试间隔
# 退避 —— 避免首次运行持续失败时每个 tick(1h)都重试、每次烧一份 LLM 配额。
attempt_dt = _parse_state_time(state.get("last_attempt_at"))
if attempt_dt is None:
return True
return datetime.now(UTC) >= attempt_dt + timedelta(hours=_CURATOR_RETRY_HOURS)
interval_hours = getattr(curator_cfg, "interval_hours", 168)
try:
last_dt = datetime.fromisoformat(last_run)
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
except Exception:
last_dt = _parse_state_time(last_run)
if last_dt is None:
# 时间串损坏 —— 视同"从未跑过",允许重跑并重写正确时间戳,而不是像旧逻辑那样
# except 返回 False 导致该用户永久不再触发。
return True
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
def manual_trigger_blocked_reason(user_id: str) -> str | None:
"""手动触发 curator 的预检。返回 None 表示可以跑;否则返回阻塞原因代码:
``evolution_disabled`` / ``curator_disabled`` / ``already_running``。
供 ``POST /api/curator/run`` 在派发后台线程**前**同步判定 —— 后台线程
(``_curator_thread_target``)本身也会再查一遍,这里只是为了让前端拿到准确反馈,
而不是无论开关状态如何都提示"已开始"。
"""
if not _skill_evolution_enabled(user_id):
return "evolution_disabled"
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
return "curator_disabled"
state = _read_curator_state(user_id)
if _run_lock_active(state, datetime.now(UTC)):
return "already_running"
return None
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
@@ -297,12 +454,13 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
if now is None:
now = datetime.now(UTC)
curator_cfg = _get_curator_config()
curator_cfg = _get_curator_config(user_id)
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
raw_entries = agent_created_entries()
skipped_referenced = 0
@@ -314,7 +472,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
if name in referenced:
skipped_referenced += 1
continue
if _is_skill_published(name):
if name in published:
skipped_published += 1
continue
eligible[name] = entry
@@ -377,9 +535,10 @@ def _render_candidate_list(user_id: str) -> str:
owned = _list_user_owned_skill_names(user_id)
referenced = _collect_referenced_skill_names()
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
entries: dict[str, dict] = {}
for k, v in all_entries().items():
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced, published_set=published):
continue
if v.get("state") == STATE_ARCHIVED:
continue
@@ -590,8 +749,72 @@ def _make_skill_archive_tool(user_id: str):
return _skill_archive
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
"""Run the curator consolidation pass as a React agent with skill tools bound."""
def _make_curator_skill_manage_tool():
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
"""
from langchain.tools import tool as lc_tool
from deerflow.tools.skill_manage_tool import _skill_manage_impl
@lc_tool("skill_manage")
def _curator_skill_manage(
action: str,
name: str,
content: str | None = None,
path: str | None = None,
find: str | None = None,
replace: str | None = None,
expected_count: int | None = None,
) -> str:
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
支持 action:create / patch / edit / write_file / remove_file / list / view。
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
Args:
action: 操作类型(见上;delete 会被拒绝)。
name: 连字符命名的技能名。
content: create / edit / write_file 的新文件内容。
path: write_file / remove_file 的支持文件路径。
find: patch 要替换的原文本。
replace: patch 的替换文本。
expected_count: patch 的预期替换次数(可选)。
"""
if action == "delete":
return (
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
)
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
return _run_async_blocking(
_skill_manage_impl(
runtime=None,
action=action,
name=name,
content=content,
path=path,
find=find,
replace=replace,
expected_count=expected_count,
)
)
return _curator_skill_manage
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
"""Run the curator consolidation pass as a React agent with skill tools bound.
返回 ``(llm_text, tool_actions)``:
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
"""
from deerflow.skills import curator_log
try:
@@ -601,10 +824,14 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.models import create_chat_model
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
model = create_chat_model()
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
curator_cfg = _get_curator_config(user_id)
# curator 评审用的模型:per-user 可配 curator.model_name(对应 config.yaml
# models[].name);为空则用主模型。
model_name = getattr(curator_cfg, "model_name", None) if curator_cfg else None
model = create_chat_model(name=model_name)
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
# (wrong action / missing required args), the exception is converted
# into a ToolMessage(status="error") so the LLM can read the error and
@@ -613,30 +840,58 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
timeout_seconds = getattr(curator_cfg, "review_timeout_seconds", 600) if curator_cfg else 600
curator_log.record(
"info",
f"LLM 评审:开始调用 React Agent(模型 {model_name or '主模型'},整体超时 {timeout_seconds}s)",
user_id=user_id,
)
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
# 给整个 React Agent 评审套一个总超时:agent.invoke 内部可能多轮调 LLM,模型层的
# 单次 request_timeout 兜不住整体。超时后本 curator 线程能正常收尾、释放运行锁,
# 而不是无限期挂起。注意:Python 无法强杀线程 —— 超时后那个 agent.invoke 线程会
# 成为孤儿(最终被模型层 request_timeout 或自然返回所终结),但不再拖住 curator。
# copy_context():把 current_user 等 ContextVar 传进工作线程,否则工具调用会把
# user_id 误解析为 DEFAULT_USER_ID。
ctx = contextvars.copy_context()
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="curator-llm")
future = pool.submit(ctx.run, agent.invoke, {"messages": [HumanMessage(content=prompt)]})
try:
result = future.result(timeout=timeout_seconds)
except concurrent.futures.TimeoutError:
curator_log.record(
"warning",
f"LLM 评审超时(超过 {timeout_seconds}s),已中止本次评审",
user_id=user_id,
)
logger.warning("Curator[%s] LLM review timed out after %ss", user_id, timeout_seconds)
return "", []
finally:
# wait=False:超时时不阻塞等待挂死的 invoke 线程。
pool.shutdown(wait=False)
messages = result.get("messages", [])
tool_actions = _extract_tool_actions(messages)
for msg in reversed(messages):
if not getattr(msg, "tool_calls", None):
content = getattr(msg, "content", "")
if isinstance(content, str) and content.strip():
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return content
return content, tool_actions
if isinstance(content, list):
text = " ".join(
block.get("text", "") if isinstance(block, dict) else str(block)
for block in content
).strip()
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return text
return text, tool_actions
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
return ""
return "", tool_actions
except Exception as e:
logger.warning("Curator LLM review failed: %s", e)
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
return ""
return "", []
def _parse_structured_summary(text: str) -> dict:
@@ -653,6 +908,44 @@ def _parse_structured_summary(text: str) -> dict:
return {"consolidations": [], "prunings": []}
def _extract_tool_actions(messages: list) -> list[dict]:
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
是否一致。每条形如 ``{tool, args, status, result}``。
"""
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
calls: dict[str, dict] = {}
for msg in messages:
for tc in getattr(msg, "tool_calls", None) or []:
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not tc_id:
continue
calls[tc_id] = {
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
}
# 再按 ToolMessage(带 tool_call_id)收集真实结果
actions: list[dict] = []
for msg in messages:
tc_id = getattr(msg, "tool_call_id", None)
if not tc_id:
continue
call = calls.get(tc_id, {})
raw = getattr(msg, "content", "")
result = raw if isinstance(raw, str) else str(raw)
actions.append(
{
"tool": call.get("tool") or getattr(msg, "name", None),
"args": call.get("args", {}),
"status": getattr(msg, "status", None) or "success",
"result": result[:500],
}
)
return actions
def run_curator_review(user_id: str) -> dict:
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
@@ -684,16 +977,23 @@ def run_curator_review(user_id: str) -> dict:
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
)
llm_text = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
if candidate_line_count == 0:
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
suggestions = {"consolidations": [], "prunings": []}
tool_actions: list[dict] = []
else:
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
# Surface per-item reasoning so operators can audit *why* each action
# was taken, not just the count.
for item in suggestions.get("consolidations", []) or []:
@@ -718,15 +1018,26 @@ def run_curator_review(user_id: str) -> dict:
user_id=user_id,
)
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
for act in tool_actions:
target = act.get("args", {}).get("name", "")
curator_log.record(
"info",
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
user_id=user_id,
)
state = _read_curator_state(user_id)
state["last_run_at"] = now.isoformat()
state["run_count"] = state.get("run_count", 0) + 1
state["last_run_summary"] = (
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
f"归档 {transition_counters['archived']} 个;"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
)
state["last_consolidation_suggestions"] = suggestions
state["last_consolidation_suggestions"] = suggestions # LLM 自述
state["last_actual_actions"] = tool_actions # 真实工具调用记录
_write_curator_state(state, user_id)
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
@@ -755,6 +1066,23 @@ def _curator_thread_target(user_id: str) -> None:
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.skills import curator_log
# 总开关 + curator 子开关:任一关闭则直接跳过。手动触发 /api/curator/run 也经由
# 本函数,因此手动触发同样受总开关与 curator.enabled 约束。
if not _skill_evolution_enabled(user_id):
logger.info("Curator[%s]: 技能进化总开关关闭,跳过", user_id)
return
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
logger.info("Curator[%s]: curator 未启用,跳过", user_id)
return
# 运行锁:防止同一用户的 curator 并发(定时器 vs 手动触发、连点手动触发)。抢不到
# 锁说明已有一轮在跑,直接跳过本次触发。
if not _try_acquire_run_lock(user_id, datetime.now(UTC)):
logger.info("Curator[%s]: 已有 curator 正在运行,跳过本次触发", user_id)
curator_log.record("info", "已有 curator 正在运行,跳过本次触发", user_id=user_id)
return
# Bind the per-thread user context for the entire curator run so every
# downstream tool / repository call resolves the correct user_id.
token = set_current_user(_CuratorUser(user_id))
@@ -765,13 +1093,16 @@ def _curator_thread_target(user_id: str) -> None:
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
finally:
reset_current_user(token)
# 无论成功或异常,必定释放运行锁。
_release_run_lock(user_id)
async def maybe_run_curator() -> None:
"""Async startup entry — kicks off a thread per user whose interval elapsed.
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
Skips users with no DB-tracked entries (a fresh deployment) and never
blocks lifespan startup on failure.
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
任何失败都不会阻断调用方。
"""
try:
from sqlalchemy import select
@@ -789,20 +1120,42 @@ async def maybe_run_curator() -> None:
logger.exception("Curator: failed to enumerate users at startup")
return
due_users: list[str] = []
for user_id in user_ids:
try:
if not should_run_now(user_id):
continue
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
thread = threading.Thread(
target=_curator_thread_target,
args=(user_id,),
daemon=True,
name=f"skill-curator-{user_id}",
)
thread.start()
if should_run_now(user_id):
due_users.append(user_id)
except Exception:
logger.exception("Curator: failed to spawn for user %s", user_id)
logger.exception("Curator: should_run_now check failed for user %s", user_id)
_dispatch_curator_runs(due_users)
def _dispatch_curator_runs(due_users: list[str]) -> None:
"""用线程池(并发上限 ``_CURATOR_MAX_CONCURRENCY``)派发到期用户的 curator 执行。
到期用户排队消费,而非一次性全部起线程齐调 LLM —— 防首次部署所有用户同时到期时
的惊群。``shutdown(wait=False)``:提交后立即返回,线程池在所有任务跑完后自行回收。
抽成独立函数便于对并发上限做单元测试。
"""
if not due_users:
return
logger.info(
"Curator: %d 个用户到期,以并发上限 %d 派发执行",
len(due_users),
_CURATOR_MAX_CONCURRENCY,
)
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=_CURATOR_MAX_CONCURRENCY,
thread_name_prefix="skill-curator",
)
try:
for user_id in due_users:
executor.submit(_curator_thread_target, user_id)
except Exception:
logger.exception("Curator: failed to submit curator tasks")
finally:
executor.shutdown(wait=False)
def get_curator_status(user_id: str) -> dict:
@@ -819,7 +1172,7 @@ def get_curator_status(user_id: str) -> dict:
s = entry.get("state", STATE_ACTIVE)
counts[s] = counts.get(s, 0) + 1
cfg = _get_curator_config()
cfg = _get_curator_config(user_id)
return {
"enabled": cfg is not None and getattr(cfg, "enabled", True),
"run_count": state.get("run_count", 0),
@@ -832,4 +1185,8 @@ def get_curator_status(user_id: str) -> dict:
"stale": counts[STATE_STALE],
"archived": counts[STATE_ARCHIVED],
},
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
or {"consolidations": [], "prunings": []},
"last_actual_actions": state.get("last_actual_actions") or [],
}
@@ -50,8 +50,10 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
try:
config = app_config or get_app_config()
model_name = config.skill_evolution.moderation_model_name
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config) if model_name else create_chat_model(thinking_enabled=False, app_config=config)
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
model_name = config.skill_evolution.curator.model_name
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config)
response = await model.ainvoke(
[
{"role": "system", "content": rubric},
@@ -6,6 +6,7 @@ import json
import logging
import os
import tempfile
import threading
from datetime import UTC, datetime
from pathlib import Path
@@ -13,6 +14,11 @@ logger = logging.getLogger(__name__)
_USAGE_FILENAME = ".usage.json"
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
# 单进程跑,故进程内锁即可保证状态流转不丢。
_usage_lock = threading.Lock()
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
@@ -75,13 +81,20 @@ def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _update_entry(name: str, updates: dict) -> None:
def _update_entry(name: str, updates) -> None:
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
避免并发下的 lost update。
"""
try:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
entry.update(updates)
data[name] = entry
_write_all(data)
with _usage_lock:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
resolved = updates(entry) if callable(updates) else updates
entry.update(resolved)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
@@ -103,18 +116,16 @@ def agent_created_entries() -> dict:
def bump_view(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
def bump_use(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
def bump_patch(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
def set_state(name: str, state: str) -> None:
@@ -132,9 +143,10 @@ def set_pinned(name: str, pinned: bool) -> None:
def forget(name: str) -> None:
try:
data = _read_all()
data.pop(name, None)
_write_all(data)
with _usage_lock:
data = _read_all()
data.pop(name, None)
_write_all(data)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
@@ -59,6 +59,66 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
return None
def _resolve_effective_user_id() -> str | None:
"""当前请求/线程绑定的用户 id;解析失败时返回 None。"""
try:
from deerflow.runtime.user_context import get_effective_user_id
return get_effective_user_id()
except Exception:
logger.debug("Failed to resolve effective user id for skill_list", exc_info=True)
return None
def _run_coro_blocking(coro):
"""在同步工具里把一个协程跑到完成。
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
执行 ``asyncio.run``,否则直接跑。
"""
import asyncio
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
用于默认 lead agent 的可见性过滤:只算"用户自己拥有的"自定义技能,不含
其他用户发布的。返回 ``None`` 表示 ``SkillStore`` 不可用(未配置数据库
等)—— 调用方应降级为"放行全部自定义技能",避免离线无库部署下技能凭空
消失。
"""
if not user_id:
return None
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = _run_coro_blocking(store.list_visible(user_id))
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
return None
@tool("skill_view")
def skill_view_tool(name: str) -> str:
"""Read the full SKILL.md content of a skill by name.
@@ -106,28 +166,47 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
under the ``skills:`` field — the agent only sees what it was wired to
use.
For the **default lead agent** (no agent_id in runtime), there is no
per-agent whitelist and every visible skill is returned.
For the **default lead agent** (no agent_id in runtime), the result is
filtered to non-archived skills that are either built-in (public) or
owned by the current user. Other users' published custom skills and
archived skills are excluded.
Returns a JSON array with fields: name, description, category, enabled,
state, use_count, view_count, patch_count, last_activity_at, pinned,
source.
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import all_entries, latest_activity_at
from deerflow.skills.types import SkillCategory
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
agent_id = _resolve_active_agent_id(runtime)
allowed = _load_agent_skill_whitelist(agent_id)
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
# 当前用户"的技能。owned 为 None 表示 SkillStore 不可用,降级放行全部自定义
# 技能(见 _load_owned_skill_names)。
owned: set[str] | None = None
if allowed is None:
owned = _load_owned_skill_names(_resolve_effective_user_id())
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=False)
usage = all_entries()
result = []
for skill in skills:
if allowed is not None and skill.name not in allowed:
continue
entry = usage.get(skill.name, {})
if allowed is not None:
# 自定义 agent:严格按 config.yaml 的 skills 白名单过滤。
if skill.name not in allowed:
continue
else:
# 默认 agent:跳过已归档技能。
if entry.get("state") == STATE_ARCHIVED:
continue
# 自定义技能仅当属于当前用户时可见;内置 public 技能始终可见。
if skill.category == SkillCategory.CUSTOM and owned is not None and skill.name not in owned:
continue
result.append({
"name": skill.name,
"description": skill.description,
@@ -84,12 +84,26 @@ def get_available_tools(
builtin_tools = BUILTIN_TOOLS.copy()
if not is_scheduled_run:
builtin_tools.append(scheduled_task_tool)
skill_evolution_config = getattr(config, "skill_evolution", None)
# skill_list / skill_view 是只读工具(发现、查看技能),与"自我进化"写能力无关,
# 始终提供给 agent。
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
builtin_tools.extend([skill_list_tool, skill_view_tool])
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
from deerflow.runtime.user_context import get_effective_user_id
skill_evolution_config = load_skill_evolution_config_for_user(get_effective_user_id())
except Exception: # noqa: BLE001 - 解析失败不能阻断 agent 构建
skill_evolution_config = getattr(config, "skill_evolution", None)
if getattr(skill_evolution_config, "enabled", False):
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
builtin_tools.append(skill_manage_tool)
# Add subagent tools only if enabled via runtime parameter
if subagent_enabled:
@@ -25,12 +25,12 @@ def test_load_overrides_returns_none_when_missing(overrides_path):
def test_save_and_load_overrides_roundtrip(overrides_path):
payload = {
"enabled": True,
"moderation_model_name": "moderation-mini",
"curator": {
"enabled": True,
"interval_hours": 12,
"stale_after_days": 7,
"archive_after_days": 30,
"model_name": "curator-mini",
},
}
written = skill_evolution_runtime.save_overrides(payload)
@@ -501,7 +501,8 @@ def test_curator_skips_published_skill(isolated_paths, monkeypatch) -> None:
monkeypatch.setattr(curator, "_list_user_owned_skill_names", lambda uid: {"public-tool"})
monkeypatch.setattr(curator, "_collect_referenced_skill_names", lambda: set())
monkeypatch.setattr(curator, "_is_skill_published", lambda name: name == "public-tool")
# curator 现在用批量接口一次性查 published 集合,而非逐技能 _is_skill_published。
monkeypatch.setattr(curator, "_list_published_skill_names", lambda: {"public-tool"})
_isolated_skills_root(isolated_paths, monkeypatch)
_seed_old_agent_skill("public-tool")
+2692 -2692
View File
File diff suppressed because it is too large. Load diff