This commit is contained in:
董宏凯 committed 2026-05-15 23:27:48 +08:00
commit e08e7be24c
404 files changed
+113007 -921

No files matched your search

@@ -8,9 +8,10 @@ from deerflow.agents.lead_agent.prompt import apply_prompt_template
from deerflow.agents.memory.summarization_hook import memory_flush_hook
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
@@ -257,6 +258,10 @@ def _build_middlewares(
resolved_app_config = app_config or get_app_config()
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
# Inject the admin prompt prefix into the model request (kept out of the
# persisted/displayed message so the user never sees it).
middlewares.append(PromptPrefixMiddleware())
# Add summarization middleware if enabled
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
if summarization_middleware is not None:
@@ -304,10 +309,10 @@ def _build_middlewares(
if custom_middlewares:
middlewares.extend(custom_middlewares)
middlewares.append(SetupAgentApprovalMiddleware())
# ClarificationMiddleware should always be last
middlewares.append(ClarificationMiddleware())
middlewares.append(SetupAgentApprovalMiddleware())
# ClarificationMiddleware should always be last
middlewares.append(ClarificationMiddleware())
return middlewares
@@ -331,9 +336,13 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
requested_model_name: str | None = cfg.get("model_name") or cfg.get("model")
is_plan_mode = cfg.get("is_plan_mode", False)
subagent_enabled = cfg.get("subagent_enabled", False)
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
writing_mode = bool(cfg.get("writing_mode", False))
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
writing_artifact_path = None
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
agent_display_name = agent_config.name if agent_config else agent_id
@@ -352,14 +361,15 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
thinking_enabled = False
logger.info(
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s",
agent_id or "default",
thinking_enabled,
reasoning_effort,
model_name,
is_plan_mode,
subagent_enabled,
max_concurrent_subagents,
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s",
agent_id or "default",
thinking_enabled,
reasoning_effort,
model_name,
is_plan_mode,
subagent_enabled,
max_concurrent_subagents,
writing_mode,
)
# Inject run metadata for LangSmith trace tagging
@@ -383,8 +393,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
"thinking_enabled": thinking_enabled,
"reasoning_effort": reasoning_effort,
"is_plan_mode": is_plan_mode,
"subagent_enabled": subagent_enabled,
"tool_groups": agent_config.tool_groups if agent_config else None,
"subagent_enabled": subagent_enabled,
"writing_mode": writing_mode,
"writing_artifact_path": writing_artifact_path,
"tool_groups": agent_config.tool_groups if agent_config else None,
"available_skills": runtime_available_skills,
}
)
@@ -403,9 +415,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
middleware=_build_middlewares(config, model_name=model_name, app_config=resolved_app_config),
system_prompt=apply_prompt_template(
subagent_enabled=subagent_enabled,
max_concurrent_subagents=max_concurrent_subagents,
available_skills=set(["bootstrap"]),
app_config=resolved_app_config,
max_concurrent_subagents=max_concurrent_subagents,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
available_skills=set(["bootstrap"]),
app_config=resolved_app_config,
),
state_schema=ThreadState,
)
@@ -424,10 +438,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
system_prompt=apply_prompt_template(
subagent_enabled=subagent_enabled,
max_concurrent_subagents=max_concurrent_subagents,
agent_id=agent_id,
agent_name=agent_display_name,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
agent_id=agent_id,
agent_name=agent_display_name,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
),
state_schema=ThreadState,
)
@@ -69,8 +69,25 @@ _enabled_skills_refresh_version = 0
_enabled_skills_refresh_event = threading.Event()
def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
"""从注入提示词的技能列表里剔除已归档(usage state=archived)的技能。
技能归档是软状态:SKILL.md 仍留在 custom/<name>/、enabled 标志不变,所以
load_skills(enabled_only=True) 仍会返回它。但归档技能不应再出现在 agent 的系统
提示词里 —— 对 agent 而言它应等同于"不存在"。失败时不过滤(返回原列表)。
"""
try:
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
entries = all_entries()
return [s for s in skills if entries.get(s.name, {}).get("state") != STATE_ARCHIVED]
except Exception:
logger.debug("Failed to filter archived skills for prompt injection", exc_info=True)
return skills
def _load_enabled_skills_sync() -> list[Skill]:
return list(get_or_new_skill_storage().load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage().load_skills(enabled_only=True)))
def _start_enabled_skills_refresh_thread() -> None:
@@ -171,7 +188,7 @@ def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[
"""
if app_config is None:
return _get_enabled_skills()
return list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True))
return _drop_archived_skills(list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True)))
def _skill_mutability_label(category: SkillCategory | str) -> str:
@@ -481,6 +498,8 @@ You: "Deploying to staging..." [proceed]
{deferred_tools_section}
{writing_mode_section}
{subagent_section}
<working_directory existed="true">
@@ -800,6 +819,8 @@ def apply_prompt_template(
*,
agent_id: str | None = None,
agent_name: str | None = None,
writing_mode: bool = False,
writing_artifact_path: str | None = None,
available_skills: set[str] | None = None,
app_config: AppConfig | None = None,
) -> str:
@@ -835,6 +856,32 @@ def apply_prompt_template(
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
current_writing_artifact = (
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
if writing_artifact_path
else ""
)
writing_mode_section = (
f"""
<writing_mode>
Writing mode is enabled for this run.
**Mandatory Output Contract**
- Treat the user's request as a writing/report-generation task.
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
{current_writing_artifact}
</writing_mode>
"""
if writing_mode
else ""
)
# Build ACP agent section only if ACP agents are configured
acp_section = _build_acp_section(app_config=app_config)
@@ -850,6 +897,7 @@ def apply_prompt_template(
soul=get_agent_soul(storage_agent_id),
skills_section=skills_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
network_policy_section=network_policy_section,
subagent_section=subagent_section,
@@ -0,0 +1,335 @@
"""Middleware that records per-LLM-call metrics for the admin audit page.
Captures **every** provider invocation regardless of the frontend speed-meter
toggle: the goal is a complete admin-side audit log of LLM usage, including
failures. The user-facing speed indicator (controlled by
``LocalSettings.display.streamSpeedMeterEnabled``) is a separate, purely
cosmetic feature — turning it off does not stop this recording.
Each call ends with one row in ``llm_call_metrics``:
- Successes record duration, token counts, and ``tokens_per_sec``.
- Errors record the duration spent before the failure, the exception class as
``error_type`` and a one-line ``error_message``.
Writes go through a best-effort store: a DB outage logs a warning and the
agent runtime keeps streaming. We intentionally place this middleware
*inside* :class:`LLMErrorHandlingMiddleware` (i.e. nested deeper in the
``wrap_model_call`` chain) so we observe the **real** provider call timings —
not the retry-wrapped successful response.
"""
from __future__ import annotations
import asyncio
import logging
import time
import uuid
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import (
ModelCallResult,
ModelRequest,
ModelResponse,
)
from langchain_core.messages import AIMessage
from langgraph.errors import GraphBubbleUp
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.llm_metrics import LlmMetricRecord, make_llm_metrics_store
from deerflow.persistence.llm_metrics.base import LlmMetricsStore
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
def _now() -> datetime:
return datetime.now(UTC)
def _safe_get_thread_id(request: ModelRequest) -> str | None:
state = getattr(request, "state", None)
if isinstance(state, dict):
thread_id = state.get("thread_id")
if isinstance(thread_id, str) and thread_id:
return thread_id
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
config = getattr(runtime, "config", None)
configurable = (config or {}).get("configurable", {}) if isinstance(config, dict) else {}
thread_id = configurable.get("thread_id") if isinstance(configurable, dict) else None
if isinstance(thread_id, str) and thread_id:
return thread_id
return None
def _safe_get_run_id(request: ModelRequest) -> str | None:
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in ("run_id", "langgraph_run_id"):
value = context.get(key)
if isinstance(value, str) and value:
return value
config = getattr(runtime, "config", None)
if isinstance(config, dict):
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
if isinstance(metadata, dict):
for key in ("run_id", "langgraph_run_id"):
value = metadata.get(key)
if isinstance(value, str) and value:
return value
return None
def _safe_get_model_name(request: ModelRequest) -> str | None:
model = getattr(request, "model", None)
for attr in ("model_name", "model", "name"):
value = getattr(model, attr, None)
if isinstance(value, str) and value:
return value
return None
def _safe_get_agent_name(request: ModelRequest) -> str | None:
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in ("agent_name", "agent_id"):
value = context.get(key)
if isinstance(value, str) and value:
return value
return None
def _coerce_int(value: Any) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
if isinstance(value, str):
try:
return int(value)
except (TypeError, ValueError):
return None
return None
def _extract_usage(response: Any) -> dict[str, int]:
"""Pull token counts out of an AIMessage, falling back to provider-native shapes.
The handler in ``wrap_model_call`` actually hands us a
:class:`langchain.agents.middleware.types.ModelResponse` dataclass whose
``.result`` is a ``list[BaseMessage]``. Earlier versions of this helper
only matched ``AIMessage`` and ``list``, which meant we silently dropped
every successful call and stored None for input/output/total. To stay
robust we now also handle ``ModelResponse`` (and ``ExtendedModelResponse``,
which wraps a ModelResponse), and we walk the messages list for the first
``AIMessage``.
LangChain normalises usage into ``message.usage_metadata`` with the keys
``input_tokens`` / ``output_tokens`` / ``total_tokens``. Several
OpenAI-compatible providers (DeepSeek, vLLM, some Azure deployments) leave
that dict partial — typically only ``total_tokens`` is populated — and the
real breakdown sits under ``response_metadata.token_usage`` with the
provider-native key names ``prompt_tokens`` / ``completion_tokens``. We
merge both so the admin audit view always has the most complete numbers
available.
"""
message: AIMessage | None = None
def _find_ai(candidate: Any) -> AIMessage | None:
if isinstance(candidate, AIMessage):
return candidate
if isinstance(candidate, list):
for item in candidate:
found = _find_ai(item)
if found is not None:
return found
return None
message = _find_ai(response)
if message is None:
# ModelResponse / ExtendedModelResponse — defined as dataclasses with a
# ``result`` field holding ``list[BaseMessage]``. Recurse via attribute
# access so we don't tightly couple to the dataclass class itself.
for attr in ("result", "model_response"):
inner = getattr(response, attr, None)
if inner is None:
continue
message = _find_ai(inner)
if message is None and attr == "model_response":
inner_result = getattr(inner, "result", None)
message = _find_ai(inner_result)
if message is not None:
break
if message is None:
return {}
out: dict[str, int] = {}
usage = getattr(message, "usage_metadata", None) or {}
if isinstance(usage, dict):
for key in ("input_tokens", "output_tokens", "total_tokens"):
coerced = _coerce_int(usage.get(key))
if coerced is not None:
out[key] = coerced
response_metadata = getattr(message, "response_metadata", None) or {}
token_usage = response_metadata.get("token_usage") if isinstance(response_metadata, dict) else None
if isinstance(token_usage, dict):
# Map both the LangChain-canonical names and OpenAI-native names; later
# writes win so a more-precise provider-native value can override a
# zero placeholder from usage_metadata.
candidates = {
"input_tokens": ("input_tokens", "prompt_tokens"),
"output_tokens": ("output_tokens", "completion_tokens"),
"total_tokens": ("total_tokens",),
}
for canonical, keys in candidates.items():
for key in keys:
coerced = _coerce_int(token_usage.get(key))
if coerced is not None and coerced > 0:
out[canonical] = coerced
break
# Derive a missing or implausibly-zero field if the other two are present.
# Some providers report ``input_tokens=0`` / ``output_tokens=0`` next to a
# real ``total_tokens`` — treat those zeros as missing so we don't silently
# store nonsense breakdowns.
def _missing_or_zero(key: str) -> bool:
return out.get(key) in (None, 0)
if "total_tokens" not in out and out.get("input_tokens") and out.get("output_tokens"):
out["total_tokens"] = out["input_tokens"] + out["output_tokens"]
if _missing_or_zero("output_tokens") and out.get("total_tokens") and out.get("input_tokens"):
derived = out["total_tokens"] - out["input_tokens"]
if derived > 0:
out["output_tokens"] = derived
if _missing_or_zero("input_tokens") and out.get("total_tokens") and out.get("output_tokens"):
derived = out["total_tokens"] - out["output_tokens"]
if derived > 0:
out["input_tokens"] = derived
return out
def _classify_exception(exc: BaseException) -> tuple[str, str]:
"""Return (error_type, error_message). Message is trimmed to 500 chars."""
name = exc.__class__.__name__
raw = (str(exc) or name).strip()
return name, (raw[:500] if len(raw) > 500 else raw)
class LlmMetricsMiddleware(AgentMiddleware[AgentState]):
"""Wrap each model call to time it and persist a metric row."""
def __init__(self, *, store: LlmMetricsStore | None = None, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._store_override = store
def _store(self) -> LlmMetricsStore | None:
if self._store_override is not None:
return self._store_override
try:
sf = get_session_factory()
except Exception:
return None
return make_llm_metrics_store(sf)
def _build_base_record(self, request: ModelRequest) -> LlmMetricRecord:
return LlmMetricRecord(
id=uuid.uuid4().hex,
created_at=_now(),
user_id=get_effective_user_id(),
thread_id=_safe_get_thread_id(request),
run_id=_safe_get_run_id(request),
agent_name=_safe_get_agent_name(request),
model_name=_safe_get_model_name(request),
)
def _finalize_success(self, record: LlmMetricRecord, response: ModelResponse, started: float) -> None:
elapsed = max(0.0, time.monotonic() - started)
record.duration_ms = int(elapsed * 1000)
usage = _extract_usage(response)
record.input_tokens = usage.get("input_tokens")
record.output_tokens = usage.get("output_tokens")
record.total_tokens = usage.get("total_tokens") or (
(record.input_tokens or 0) + (record.output_tokens or 0) or None
)
if record.output_tokens is not None and elapsed > 0:
record.tokens_per_sec = round(record.output_tokens / elapsed, 3)
record.status = "success"
def _finalize_error(self, record: LlmMetricRecord, exc: BaseException, started: float) -> None:
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
record.status = "error"
record.error_type, record.error_message = _classify_exception(exc)
def _flush(self, record: LlmMetricRecord) -> None:
store = self._store()
if store is None:
return
try:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
# Fire-and-forget — never let metric IO block the request path.
loop.create_task(store.record(record))
else:
asyncio.run(store.record(record))
except Exception:
logger.warning("LlmMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
record = self._build_base_record(request)
started = time.monotonic()
try:
response = handler(request)
except GraphBubbleUp:
# LangGraph control flow — not a real LLM failure.
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_success(record, response, started)
self._flush(record)
return response
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
record = self._build_base_record(request)
started = time.monotonic()
try:
response = await handler(request)
except GraphBubbleUp:
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_success(record, response, started)
self._flush(record)
return response
@@ -0,0 +1,78 @@
"""Middleware that injects the admin-configured prompt prefix at model-call time.
The prompt-prefix feature lets an admin prepend a fixed instruction to the
user's question before it reaches the LLM. To keep the prefix invisible in the
chat UI, the Gateway does **not** concatenate it into the persisted
``HumanMessage.content``; instead it tags the first human message with
``additional_kwargs["prompt_prefix"]`` (see ``app.gateway.services``).
This middleware reads that tag and prepends the prefix to the message content
*only in the request handed to the model*. The persisted state — and therefore
what the frontend renders — keeps the clean, prefix-free content.
"""
from collections.abc import Awaitable, Callable
from typing import override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import HumanMessage
PROMPT_PREFIX_KEY = "prompt_prefix"
class PromptPrefixMiddleware(AgentMiddleware[AgentState]):
"""Prepends ``additional_kwargs['prompt_prefix']`` to human messages for the LLM only.
The transformation is applied to the model request copy, never written back
to state, so it is naturally idempotent across the multiple model calls of a
single turn.
"""
@staticmethod
def _apply(messages: list) -> list | None:
"""Return a patched copy with prefixes prepended, or None if nothing to do."""
patched: list | None = None
for idx, msg in enumerate(messages):
if not isinstance(msg, HumanMessage):
continue
raw_prefix = (getattr(msg, "additional_kwargs", None) or {}).get(PROMPT_PREFIX_KEY)
prefix = str(raw_prefix).strip() if raw_prefix else ""
if not prefix:
continue
content = msg.content
if isinstance(content, str):
new_content: object = f"{prefix}\n\n{content}".strip("\n")
elif isinstance(content, list):
# Structured content (text blocks + media): prepend a text block.
new_content = [{"type": "text", "text": prefix}, *content]
else:
new_content = f"{prefix}\n\n{content!s}"
if patched is None:
patched = list(messages)
# model_copy preserves the message id so streaming dedup is unaffected.
patched[idx] = msg.model_copy(update={"content": new_content})
return patched
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
patched = self._apply(request.messages)
if patched is not None:
request = request.override(messages=patched)
return handler(request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
patched = self._apply(request.messages)
if patched is not None:
request = request.override(messages=patched)
return await handler(request)
@@ -72,7 +72,7 @@ class SetupAgentApprovalMiddleware(AgentMiddleware[SetupAgentApprovalMiddlewareS
approval_msg = ToolMessage(
id=self._stable_message_id(tool_call_id, args),
content="请确认是否创建/更新这个专家配置。",
content="请确认是否创建/更新这个智能体配置。",
tool_call_id=tool_call_id,
name="setup_agent",
additional_kwargs={"setup_agent_approval": payload},
@@ -96,6 +96,14 @@ def _build_runtime_middlewares(
middlewares.append(LLMErrorHandlingMiddleware(app_config=app_config))
# LLM call metrics — always on, even when the user-facing speed indicator
# is disabled. We append *after* the retry middleware so the metrics
# middleware sits deeper in the wrap chain and observes individual provider
# attempts (including the failed ones).
from deerflow.agents.middlewares.llm_metrics_middleware import LlmMetricsMiddleware
middlewares.append(LlmMetricsMiddleware())
# Guardrail middleware (if configured)
guardrails_config = app_config.guardrails
if guardrails_config.enabled and guardrails_config.provider:
@@ -45,7 +45,12 @@ class McpServerConfig(BaseModel):
headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers to send (for sse or http type)")
oauth: McpOAuthConfig | None = Field(default=None, description="OAuth configuration (for sse or http type)")
description: str = Field(default="", description="Human-readable description of what this MCP server provides")
model_config = ConfigDict(extra="allow")
disabled_tools: list[str] = Field(
default_factory=list,
description="List of tool names (without server-name prefix) to hide from the agent",
alias="disabledTools",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
@@ -0,0 +1,77 @@
"""System-wide runtime settings editable from the admin UI.
Backed by a single JSON file under the DeerFlow runtime home (default
``.deer-flow/system_settings.json``). Unlike :mod:`extensions_config` this
file is **not** part of the project source tree — it is per-deployment
state, and it stays writable at runtime.
The only feature stored here at the moment is the **prompt prefix toggle**
shown in the chat input: an administrator can configure a label and a
prefix string, and any user can flip the toggle on/off; when enabled, the
prefix is prepended to the first human message of a new thread.
"""
from __future__ import annotations
import json
import logging
import threading
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.runtime_paths import runtime_home
logger = logging.getLogger(__name__)
_SETTINGS_FILE = "system_settings.json"
_lock = threading.Lock()
class PromptPrefixSettings(BaseModel):
"""Admin-configurable first-message prompt prefix."""
enabled: bool = Field(default=False, description="Whether the prefix toggle is offered to users at all")
switch_label: str = Field(default="加入立场前缀", description="Toggle label shown in the chat input")
prompt_prefix: str = Field(default="", description="Text prepended to the first user message when enabled")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
model_config = ConfigDict(extra="ignore")
def _settings_path() -> Path:
home = runtime_home()
home.mkdir(parents=True, exist_ok=True)
return home / _SETTINGS_FILE
def load_system_settings() -> SystemSettings:
"""Read the on-disk system settings, returning defaults when missing."""
path = _settings_path()
if not path.is_file():
return SystemSettings()
try:
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
logger.warning("System settings file %s is not a JSON object; ignoring", path)
return SystemSettings()
return SystemSettings.model_validate(raw)
except Exception as exc:
logger.warning("Failed to read system settings from %s: %s", path, exc)
return SystemSettings()
def save_system_settings(settings: SystemSettings) -> SystemSettings:
"""Persist the settings document atomically, returning the saved value."""
path = _settings_path()
payload = settings.model_dump(mode="json")
with _lock:
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
tmp.replace(path)
return settings
@@ -121,6 +121,25 @@ async def get_mcp_tools() -> list[BaseTool]:
# Get all tools from all servers
tools = await client.get_tools()
# Filter out tools listed in each server's `disabled_tools`. Tool names from
# MultiServerMCPClient are prefixed with the server name (because we pass
# `tool_name_prefix=True`), so we compare against `<server>_<tool>` here.
disabled_full_names: set[str] = set()
for server_name, server_cfg in extensions_config.get_enabled_mcp_servers().items():
for tool_name in server_cfg.disabled_tools or []:
name = (tool_name or "").strip()
if not name:
continue
disabled_full_names.add(f"{server_name}_{name}")
disabled_full_names.add(name)
if disabled_full_names:
kept = [t for t in tools if t.name not in disabled_full_names]
removed = len(tools) - len(kept)
if removed:
logger.info(f"Filtered out {removed} disabled MCP tool(s)")
tools = kept
logger.info(f"Successfully loaded {len(tools)} tool(s) from MCP servers")
# Patch tools to support sync invocation, as deerflow client streams synchronously
@@ -0,0 +1,25 @@
"""Persistence layer for per-LLM-call metrics.
Each completed LLM invocation (success or failure) writes a single row so
administrators can audit usage and speed. Designed to be cheap to write —
the middleware fires once per provider call and never blocks streaming.
"""
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
__all__ = [
"LlmCallMetricRow",
"LlmMetricRecord",
"LlmMetricsStore",
"SqlLlmMetricsStore",
"make_llm_metrics_store",
]
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlLlmMetricsStore(session_factory)
@@ -0,0 +1,66 @@
"""Abstract repository contract for LLM call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class LlmMetricRecord:
"""In-memory shape of a metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
model_name: str | None = None
duration_ms: int = 0
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
tokens_per_sec: float | None = None
status: str = "success"
error_type: str | None = None
error_message: str | None = None
@dataclass
class LlmMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
model_name: str | None = None
status: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class LlmMetricsStore(ABC):
"""Persistence contract for the metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: LlmMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: LlmMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: LlmMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@@ -0,0 +1,62 @@
"""ORM model for per-LLM-call metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class LlmCallMetricRow(Base):
"""One row per completed (or failed) LLM provider invocation."""
__tablename__ = "llm_call_metrics"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
# sort/filter by recency.
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
# Wall-clock duration of the provider invocation in milliseconds. Always set.
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
input_tokens: Mapped[int | None] = mapped_column(Integer)
output_tokens: Mapped[int | None] = mapped_column(Integer)
total_tokens: Mapped[int | None] = mapped_column(Integer)
# Output tokens / seconds; null when there is no completion (errors).
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
# "success" or "error". Errors store the classification and original message
# so admins can scan for transient vs auth/quota issues.
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
error_type: Mapped[str | None] = mapped_column(String(64))
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"model_name": self.model_name,
"duration_ms": self.duration_ms,
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"total_tokens": self.total_tokens,
"tokens_per_sec": self.tokens_per_sec,
"status": self.status,
"error_type": self.error_type,
"error_message": self.error_message,
}
@@ -0,0 +1,109 @@
"""SQLAlchemy-backed LLM metrics repository."""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
logger = logging.getLogger(__name__)
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
return row.to_dict()
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
class SqlLlmMetricsStore(LlmMetricsStore):
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: LlmMetricRecord) -> None:
row = LlmCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
model_name=metric.model_name,
duration_ms=metric.duration_ms,
input_tokens=metric.input_tokens,
output_tokens=metric.output_tokens,
total_tokens=metric.total_tokens,
tokens_per_sec=metric.tokens_per_sec,
status=metric.status,
error_type=metric.error_type,
error_message=metric.error_message,
)
try:
async with self._sf() as session:
session.add(row)
await session.commit()
except Exception:
# Metrics are best-effort. Never let a persistence failure surface
# to the agent runtime — log and move on.
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: LlmMetricsQuery):
if query.user_id:
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
if query.model_name:
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
if query.status:
stmt = stmt.where(LlmCallMetricRow.status == query.status)
if query.since is not None:
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
stmt = select(LlmCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def count(self, query: LlmMetricsQuery) -> int:
stmt = select(func.count()).select_from(LlmCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
# Page through the result so the Excel export stays bounded in memory
# for large date windows.
page_size = 500
offset = 0
while True:
page_query = LlmMetricsQuery(
user_id=query.user_id,
model_name=query.model_name,
status=query.status,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
page = await self.list(page_query)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
@@ -0,0 +1,59 @@
"""Create llm_call_metrics table.
Revision ID: 20260514_02
Revises: 20260514_01
Create Date: 2026-05-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260514_02"
down_revision = "20260514_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("llm_call_metrics"):
return
op.create_table(
"llm_call_metrics",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("thread_id", sa.String(64), nullable=True),
sa.Column("run_id", sa.String(64), nullable=True),
sa.Column("agent_name", sa.String(128), nullable=True),
sa.Column("model_name", sa.String(128), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
sa.Column("input_tokens", sa.Integer(), nullable=True),
sa.Column("output_tokens", sa.Integer(), nullable=True),
sa.Column("total_tokens", sa.Integer(), nullable=True),
sa.Column("tokens_per_sec", sa.Float(), nullable=True),
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
sa.Column("error_type", sa.String(64), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_llm_call_metrics_created_at", "llm_call_metrics", ["created_at"])
op.create_index("ix_llm_call_metrics_user_id", "llm_call_metrics", ["user_id"])
op.create_index("ix_llm_call_metrics_model_name", "llm_call_metrics", ["model_name"])
op.create_index("ix_llm_call_metrics_status", "llm_call_metrics", ["status"])
op.create_index("ix_llm_call_metrics_thread_id", "llm_call_metrics", ["thread_id"])
op.create_index("ix_llm_call_metrics_run_id", "llm_call_metrics", ["run_id"])
op.create_index("ix_llm_call_metrics_agent_name", "llm_call_metrics", ["agent_name"])
def downgrade() -> None:
if _has_table("llm_call_metrics"):
op.drop_table("llm_call_metrics")
@@ -0,0 +1,60 @@
"""Create tags and tag_assignments tables.
Revision ID: 20260515_01
Revises: 20260514_02
Create Date: 2026-05-15
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260515_01"
down_revision = "20260514_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("tags"):
op.create_table(
"tags",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("tag_type", sa.String(32), nullable=False),
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),
)
op.create_index("ix_tags_tag_type", "tags", ["tag_type"])
if not _has_table("tag_assignments"):
op.create_table(
"tag_assignments",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("tag_id", sa.String(64), nullable=False),
sa.Column("target_type", sa.String(32), nullable=False),
sa.Column("target_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
)
op.create_index("ix_tag_assignments_tag_id", "tag_assignments", ["tag_id"])
op.create_index("ix_tag_assignments_target_type", "tag_assignments", ["target_type"])
op.create_index("ix_tag_assignments_target_id", "tag_assignments", ["target_id"])
def downgrade() -> None:
if _has_table("tag_assignments"):
op.drop_table("tag_assignments")
if _has_table("tags"):
op.drop_table("tags")
@@ -17,8 +17,10 @@ there is no matching entity directory.
from deerflow.persistence.agents.model import AgentRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import (
ScheduledTaskDeliveryProfileRow,
@@ -28,13 +30,16 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.user.model import UserRow
__all__ = [
"FeedbackRow",
"AgentRow",
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",
"RunEventRow",
"RunRow",
"ScheduledTaskDeliveryProfileRow",
@@ -43,6 +48,8 @@ __all__ = [
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SkillRow",
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"UserRow",
]
@@ -0,0 +1,14 @@
"""Persistence layer for tags and tag-to-resource assignments."""
from deerflow.persistence.tags.base import TagStore
from deerflow.persistence.tags.memory import MemoryTagStore
from deerflow.persistence.tags.model import TAG_TYPES
from deerflow.persistence.tags.sql import TagRepository
__all__ = ["TagStore", "MemoryTagStore", "TagRepository", "TAG_TYPES", "make_tag_store"]
def make_tag_store(session_factory):
if session_factory is None:
return MemoryTagStore()
return TagRepository(session_factory)
@@ -0,0 +1,73 @@
"""Abstract persistence API for tags and tag-to-resource assignments."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class TagStore(ABC):
# ------------------------------------------------------------------ tags
@abstractmethod
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
"""List tags, optionally filtered by ``tag_type``.
Each record includes a derived ``usage_count`` (number of assignments).
"""
raise NotImplementedError
@abstractmethod
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create a tag. ``data`` must carry ``id``, ``name``, ``tag_type``.
Raises ``ValueError`` if ``(tag_type, name)`` is not unique.
"""
raise NotImplementedError
@abstractmethod
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update ``name`` / ``color`` / ``description``. ``tag_type`` is immutable.
Returns ``None`` if the tag does not exist. Raises ``ValueError`` on a
name collision within the type.
"""
raise NotImplementedError
@abstractmethod
async def delete_tag(self, tag_id: str) -> bool:
"""Delete a tag and cascade-remove all of its assignments."""
raise NotImplementedError
# ----------------------------------------------------------- assignments
@abstractmethod
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
"""Return the tags currently assigned to one resource."""
raise NotImplementedError
@abstractmethod
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
"""Replace the full set of tags on a resource.
Every id in ``tag_ids`` must reference an existing tag whose
``tag_type`` equals ``target_type``; otherwise raises ``ValueError``.
"""
raise NotImplementedError
@abstractmethod
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
"""Bulk-fetch tags for many resources at once (avoids N+1 on list pages).
Returns ``{target_id: [tag, ...]}`` for ids that have at least one tag.
"""
raise NotImplementedError
@abstractmethod
async def unassign_all(self, target_type: str, target_id: str) -> int:
"""Drop every tag assignment for a resource. Returns the count removed."""
raise NotImplementedError
@@ -0,0 +1,138 @@
"""In-memory ``TagStore`` implementation for tests."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.tags.base import TagStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(record: dict[str, Any]) -> dict[str, Any]:
return copy.deepcopy(record)
class MemoryTagStore(TagStore):
def __init__(self) -> None:
self._tags: dict[str, dict[str, Any]] = {}
# assignments keyed by surrogate id
self._assignments: dict[str, dict[str, Any]] = {}
# ------------------------------------------------------------------ tags
def _usage_count(self, tag_id: str) -> int:
return sum(1 for a in self._assignments.values() if a["tag_id"] == tag_id)
def _with_count(self, record: dict[str, Any]) -> dict[str, Any]:
out = _copy(record)
out["usage_count"] = self._usage_count(record["id"])
return out
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
rows = [r for r in self._tags.values() if tag_type is None or r["tag_type"] == tag_type]
rows.sort(key=lambda r: r["created_at"])
return [self._with_count(r) for r in rows]
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
record = self._tags.get(tag_id)
return self._with_count(record) if record is not None else None
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
tag_id = data["id"]
name = data["name"]
tag_type = data["tag_type"]
if any(r["tag_type"] == tag_type and r["name"] == name for r in self._tags.values()):
raise ValueError(f"Tag '{name}' already exists for type '{tag_type}'")
record = {
"id": tag_id,
"name": name,
"tag_type": tag_type,
"color": data.get("color") or "gray",
"description": data.get("description") or "",
"created_at": _now(),
"updated_at": _now(),
}
self._tags[tag_id] = record
return self._with_count(record)
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
record = self._tags.get(tag_id)
if record is None:
return None
new_name = data.get("name", record["name"])
if any(r["id"] != tag_id and r["tag_type"] == record["tag_type"] and r["name"] == new_name for r in self._tags.values()):
raise ValueError(f"Tag '{new_name}' already exists for type '{record['tag_type']}'")
if "name" in data:
record["name"] = data["name"]
if "color" in data:
record["color"] = data["color"]
if "description" in data:
record["description"] = data["description"]
record["updated_at"] = _now()
return self._with_count(record)
async def delete_tag(self, tag_id: str) -> bool:
if tag_id not in self._tags:
return False
del self._tags[tag_id]
for aid in [aid for aid, a in self._assignments.items() if a["tag_id"] == tag_id]:
del self._assignments[aid]
return True
# ----------------------------------------------------------- assignments
def _tags_for(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
tag_ids = [a["tag_id"] for a in self._assignments.values() if a["target_type"] == target_type and a["target_id"] == target_id]
records = [self._tags[tid] for tid in tag_ids if tid in self._tags]
records.sort(key=lambda r: r["name"])
return [self._with_count(r) for r in records]
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
return self._tags_for(target_type, target_id)
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
unique_ids = list(dict.fromkeys(tag_ids))
for tid in unique_ids:
tag = self._tags.get(tid)
if tag is None:
raise ValueError(f"Tag '{tid}' does not exist")
if tag["tag_type"] != target_type:
raise ValueError(f"Tag '{tid}' is type '{tag['tag_type']}', cannot tag a '{target_type}' resource")
# drop existing, then re-add
for aid in [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]:
del self._assignments[aid]
for tid in unique_ids:
aid = f"{tid}:{target_type}:{target_id}"
self._assignments[aid] = {
"id": aid,
"tag_id": tid,
"target_type": target_type,
"target_id": target_id,
"created_at": _now(),
}
return self._tags_for(target_type, target_id)
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
wanted = set(target_ids)
out: dict[str, list[dict[str, Any]]] = {}
for a in self._assignments.values():
if a["target_type"] != target_type or a["target_id"] not in wanted:
continue
tag = self._tags.get(a["tag_id"])
if tag is None:
continue
out.setdefault(a["target_id"], []).append(self._with_count(tag))
for tags in out.values():
tags.sort(key=lambda r: r["name"])
return out
async def unassign_all(self, target_type: str, target_id: str) -> int:
victims = [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]
for aid in victims:
del self._assignments[aid]
return len(victims)
@@ -0,0 +1,47 @@
"""ORM models for tags and tag-to-resource assignments.
Two tables:
- ``tags`` — a tag belongs to exactly one ``tag_type`` (``agent`` / ``skill`` /
``scheduled_task``). Tag names are unique *within* a type.
- ``tag_assignments`` — many-to-many link between a tag and a target resource.
``target_type`` is denormalized from the tag so the table is queryable by
``(target_type, target_id)`` without a join. The API layer enforces the
invariant ``target_type == tag.tag_type``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
# Allowed tag/target types. Kept here so persistence + API share one source.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
class TagRow(Base):
__tablename__ = "tags"
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class TagAssignmentRow(Base):
__tablename__ = "tag_assignments"
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,187 @@
"""SQLAlchemy-backed tag + tag-assignment repository."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tags.base import TagStore
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
def _tag_to_dict(row: TagRow, usage_count: int = 0) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
data["usage_count"] = usage_count
return data
class TagRepository(TagStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
async def _usage_counts(session: AsyncSession, tag_ids: list[str]) -> dict[str, int]:
if not tag_ids:
return {}
stmt = select(TagAssignmentRow.tag_id, func.count()).where(TagAssignmentRow.tag_id.in_(tag_ids)).group_by(TagAssignmentRow.tag_id)
result = await session.execute(stmt)
return {tag_id: count for tag_id, count in result.all()}
# ------------------------------------------------------------------ tags
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
stmt = select(TagRow)
if tag_type is not None:
stmt = stmt.where(TagRow.tag_type == tag_type)
stmt = stmt.order_by(TagRow.created_at.asc())
async with self._sf() as session:
rows = list((await session.execute(stmt)).scalars())
counts = await self._usage_counts(session, [r.id for r in rows])
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return None
counts = await self._usage_counts(session, [tag_id])
return _tag_to_dict(row, counts.get(tag_id, 0))
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = TagRow(
id=data["id"],
name=data["name"],
tag_type=data["tag_type"],
color=data.get("color") or "gray",
description=data.get("description") or "",
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise ValueError(f"Tag '{data['name']}' already exists for type '{data['tag_type']}'") from exc
await session.refresh(row)
return _tag_to_dict(row, 0)
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return None
if "name" in data:
row.name = data["name"]
if "color" in data:
row.color = data["color"]
if "description" in data:
row.description = data["description"]
row.updated_at = datetime.now(UTC)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise ValueError(f"Tag '{data.get('name')}' already exists for type '{row.tag_type}'") from exc
await session.refresh(row)
counts = await self._usage_counts(session, [tag_id])
return _tag_to_dict(row, counts.get(tag_id, 0))
async def delete_tag(self, tag_id: str) -> bool:
async with self._sf() as session:
row = await session.get(TagRow, tag_id)
if row is None:
return False
await session.execute(delete(TagAssignmentRow).where(TagAssignmentRow.tag_id == tag_id))
await session.delete(row)
await session.commit()
return True
# ----------------------------------------------------------- assignments
async def _tags_for(self, session: AsyncSession, target_type: str, target_id: str) -> list[dict[str, Any]]:
stmt = (
select(TagRow)
.join(TagAssignmentRow, TagAssignmentRow.tag_id == TagRow.id)
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id == target_id)
.order_by(TagRow.name.asc())
)
rows = list((await session.execute(stmt)).scalars())
counts = await self._usage_counts(session, [r.id for r in rows])
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
return await self._tags_for(session, target_type, target_id)
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
unique_ids = list(dict.fromkeys(tag_ids))
async with self._sf() as session:
if unique_ids:
found = list((await session.execute(select(TagRow).where(TagRow.id.in_(unique_ids)))).scalars())
by_id = {r.id: r for r in found}
for tid in unique_ids:
tag = by_id.get(tid)
if tag is None:
raise ValueError(f"Tag '{tid}' does not exist")
if tag.tag_type != target_type:
raise ValueError(f"Tag '{tid}' is type '{tag.tag_type}', cannot tag a '{target_type}' resource")
await session.execute(
delete(TagAssignmentRow).where(
TagAssignmentRow.target_type == target_type,
TagAssignmentRow.target_id == target_id,
)
)
now = datetime.now(UTC)
for tid in unique_ids:
session.add(
TagAssignmentRow(
id=uuid.uuid4().hex,
tag_id=tid,
target_type=target_type,
target_id=target_id,
created_at=now,
)
)
await session.commit()
return await self._tags_for(session, target_type, target_id)
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
if not target_ids:
return {}
stmt = (
select(TagAssignmentRow.target_id, TagRow)
.join(TagRow, TagAssignmentRow.tag_id == TagRow.id)
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id.in_(target_ids))
.order_by(TagRow.name.asc())
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
tag_rows = [tag for _, tag in rows]
counts = await self._usage_counts(session, [t.id for t in tag_rows])
out: dict[str, list[dict[str, Any]]] = {}
for target_id, tag in rows:
out.setdefault(target_id, []).append(_tag_to_dict(tag, counts.get(tag.id, 0)))
return out
async def unassign_all(self, target_type: str, target_id: str) -> int:
async with self._sf() as session:
result = await session.execute(
delete(TagAssignmentRow).where(
TagAssignmentRow.target_type == target_type,
TagAssignmentRow.target_id == target_id,
)
)
await session.commit()
return result.rowcount or 0
@@ -247,7 +247,11 @@ class ScheduledTaskService:
accessible = await self._accessible_task(user_id, task_id)
if accessible is None:
return []
return await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
runs = await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
can_edit = accessible.get("role") == "owner"
for run in runs:
run["can_edit"] = can_edit
return runs
async def get_run(self, user_id: str, run_id: str) -> dict[str, Any] | None:
run = await self.store.get_run(run_id, user_id)
@@ -316,7 +320,7 @@ class ScheduledTaskService:
except ValueError as exc:
raise PermissionError("Refusing to edit file outside the run archive") from exc
archive_path.write_text(new_content, encoding="utf-8")
archive_path.write_text(new_content, encoding="utf-8", newline="")
new_size = archive_path.stat().st_size
# Reflect the new size in the run row so subsequent list/preview calls
@@ -342,6 +346,10 @@ class ScheduledTaskService:
shutil.rmtree(archive_dir)
except Exception:
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
return True
candidate = await self.store.get_run_any(run_id)
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
raise PermissionError("Only the run owner may delete scheduled task runs")
return deleted
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
@@ -264,7 +264,7 @@ def _eligible_for_curation(
def should_run_now(user_id: str) -> bool:
"""Return True if the curator interval has elapsed for ``user_id`` and it
is not paused."""
curator_cfg = _get_curator_config()
curator_cfg = _get_curator_config(user_id)
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
return False
state = _read_curator_state(user_id)
@@ -278,7 +278,7 @@ def should_run_now(user_id: str) -> bool:
last_dt = datetime.fromisoformat(last_run)
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
except Exception:
return True
return False
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
@@ -297,7 +297,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
if now is None:
now = datetime.now(UTC)
curator_cfg = _get_curator_config()
curator_cfg = _get_curator_config(user_id)
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
@@ -590,8 +590,72 @@ def _make_skill_archive_tool(user_id: str):
return _skill_archive
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
"""Run the curator consolidation pass as a React agent with skill tools bound."""
def _make_curator_skill_manage_tool():
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
"""
from langchain.tools import tool as lc_tool
from deerflow.tools.skill_manage_tool import _skill_manage_impl
@lc_tool("skill_manage")
def _curator_skill_manage(
action: str,
name: str,
content: str | None = None,
path: str | None = None,
find: str | None = None,
replace: str | None = None,
expected_count: int | None = None,
) -> str:
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
支持 action:create / patch / edit / write_file / remove_file / list / view。
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
Args:
action: 操作类型(见上;delete 会被拒绝)。
name: 连字符命名的技能名。
content: create / edit / write_file 的新文件内容。
path: write_file / remove_file 的支持文件路径。
find: patch 要替换的原文本。
replace: patch 的替换文本。
expected_count: patch 的预期替换次数(可选)。
"""
if action == "delete":
return (
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
)
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
return _run_async_blocking(
_skill_manage_impl(
runtime=None,
action=action,
name=name,
content=content,
path=path,
find=find,
replace=replace,
expected_count=expected_count,
)
)
return _curator_skill_manage
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
"""Run the curator consolidation pass as a React agent with skill tools bound.
返回 ``(llm_text, tool_actions)``:
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
"""
from deerflow.skills import curator_log
try:
@@ -601,10 +665,10 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
from deerflow.models import create_chat_model
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
model = create_chat_model()
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
# (wrong action / missing required args), the exception is converted
# into a ToolMessage(status="error") so the LLM can read the error and
@@ -618,25 +682,26 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
messages = result.get("messages", [])
tool_actions = _extract_tool_actions(messages)
for msg in reversed(messages):
if not getattr(msg, "tool_calls", None):
content = getattr(msg, "content", "")
if isinstance(content, str) and content.strip():
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return content
return content, tool_actions
if isinstance(content, list):
text = " ".join(
block.get("text", "") if isinstance(block, dict) else str(block)
for block in content
).strip()
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
return text
return text, tool_actions
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
return ""
return "", tool_actions
except Exception as e:
logger.warning("Curator LLM review failed: %s", e)
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
return ""
return "", []
def _parse_structured_summary(text: str) -> dict:
@@ -653,6 +718,44 @@ def _parse_structured_summary(text: str) -> dict:
return {"consolidations": [], "prunings": []}
def _extract_tool_actions(messages: list) -> list[dict]:
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
是否一致。每条形如 ``{tool, args, status, result}``。
"""
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
calls: dict[str, dict] = {}
for msg in messages:
for tc in getattr(msg, "tool_calls", None) or []:
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not tc_id:
continue
calls[tc_id] = {
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
}
# 再按 ToolMessage(带 tool_call_id)收集真实结果
actions: list[dict] = []
for msg in messages:
tc_id = getattr(msg, "tool_call_id", None)
if not tc_id:
continue
call = calls.get(tc_id, {})
raw = getattr(msg, "content", "")
result = raw if isinstance(raw, str) else str(raw)
actions.append(
{
"tool": call.get("tool") or getattr(msg, "name", None),
"args": call.get("args", {}),
"status": getattr(msg, "status", None) or "success",
"result": result[:500],
}
)
return actions
def run_curator_review(user_id: str) -> dict:
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
@@ -684,16 +787,23 @@ def run_curator_review(user_id: str) -> dict:
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
)
llm_text = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
if candidate_line_count == 0:
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
suggestions = {"consolidations": [], "prunings": []}
tool_actions: list[dict] = []
else:
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
suggestions = _parse_structured_summary(llm_text)
curator_log.record(
"info",
(
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
f"建议删减 {len(suggestions['prunings'])} 项"
),
user_id=user_id,
)
# Surface per-item reasoning so operators can audit *why* each action
# was taken, not just the count.
for item in suggestions.get("consolidations", []) or []:
@@ -718,15 +828,26 @@ def run_curator_review(user_id: str) -> dict:
user_id=user_id,
)
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
for act in tool_actions:
target = act.get("args", {}).get("name", "")
curator_log.record(
"info",
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
user_id=user_id,
)
state = _read_curator_state(user_id)
state["last_run_at"] = now.isoformat()
state["run_count"] = state.get("run_count", 0) + 1
state["last_run_summary"] = (
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
f"归档 {transition_counters['archived']} 个;"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
)
state["last_consolidation_suggestions"] = suggestions
state["last_consolidation_suggestions"] = suggestions # LLM 自述
state["last_actual_actions"] = tool_actions # 真实工具调用记录
_write_curator_state(state, user_id)
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
@@ -768,10 +889,11 @@ def _curator_thread_target(user_id: str) -> None:
async def maybe_run_curator() -> None:
"""Async startup entry — kicks off a thread per user whose interval elapsed.
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
Skips users with no DB-tracked entries (a fresh deployment) and never
blocks lifespan startup on failure.
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
任何失败都不会阻断调用方。
"""
try:
from sqlalchemy import select
@@ -819,7 +941,7 @@ def get_curator_status(user_id: str) -> dict:
s = entry.get("state", STATE_ACTIVE)
counts[s] = counts.get(s, 0) + 1
cfg = _get_curator_config()
cfg = _get_curator_config(user_id)
return {
"enabled": cfg is not None and getattr(cfg, "enabled", True),
"run_count": state.get("run_count", 0),
@@ -832,4 +954,8 @@ def get_curator_status(user_id: str) -> dict:
"stale": counts[STATE_STALE],
"archived": counts[STATE_ARCHIVED],
},
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
or {"consolidations": [], "prunings": []},
"last_actual_actions": state.get("last_actual_actions") or [],
}
@@ -6,6 +6,7 @@ import json
import logging
import os
import tempfile
import threading
from datetime import UTC, datetime
from pathlib import Path
@@ -13,6 +14,11 @@ logger = logging.getLogger(__name__)
_USAGE_FILENAME = ".usage.json"
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
# 单进程跑,故进程内锁即可保证状态流转不丢。
_usage_lock = threading.Lock()
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
@@ -75,13 +81,20 @@ def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _update_entry(name: str, updates: dict) -> None:
def _update_entry(name: str, updates) -> None:
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
避免并发下的 lost update。
"""
try:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
entry.update(updates)
data[name] = entry
_write_all(data)
with _usage_lock:
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
resolved = updates(entry) if callable(updates) else updates
entry.update(resolved)
data[name] = entry
_write_all(data)
except Exception:
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
@@ -103,18 +116,16 @@ def agent_created_entries() -> dict:
def bump_view(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
def bump_use(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
def bump_patch(name: str) -> None:
entry = get_entry(name)
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
def set_state(name: str, state: str) -> None:
@@ -132,9 +143,10 @@ def set_pinned(name: str, pinned: bool) -> None:
def forget(name: str) -> None:
try:
data = _read_all()
data.pop(name, None)
_write_all(data)
with _usage_lock:
data = _read_all()
data.pop(name, None)
_write_all(data)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
@@ -59,6 +59,66 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
return None
def _resolve_effective_user_id() -> str | None:
"""当前请求/线程绑定的用户 id;解析失败时返回 None。"""
try:
from deerflow.runtime.user_context import get_effective_user_id
return get_effective_user_id()
except Exception:
logger.debug("Failed to resolve effective user id for skill_list", exc_info=True)
return None
def _run_coro_blocking(coro):
"""在同步工具里把一个协程跑到完成。
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
执行 ``asyncio.run``,否则直接跑。
"""
import asyncio
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
用于默认 lead agent 的可见性过滤:只算"用户自己拥有的"自定义技能,不含
其他用户发布的。返回 ``None`` 表示 ``SkillStore`` 不可用(未配置数据库
等)—— 调用方应降级为"放行全部自定义技能",避免离线无库部署下技能凭空
消失。
"""
if not user_id:
return None
try:
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills import make_skill_store
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = _run_coro_blocking(store.list_visible(user_id))
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
return None
@tool("skill_view")
def skill_view_tool(name: str) -> str:
"""Read the full SKILL.md content of a skill by name.
@@ -106,28 +166,47 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
under the ``skills:`` field — the agent only sees what it was wired to
use.
For the **default lead agent** (no agent_id in runtime), there is no
per-agent whitelist and every visible skill is returned.
For the **default lead agent** (no agent_id in runtime), the result is
filtered to non-archived skills that are either built-in (public) or
owned by the current user. Other users' published custom skills and
archived skills are excluded.
Returns a JSON array with fields: name, description, category, enabled,
state, use_count, view_count, patch_count, last_activity_at, pinned,
source.
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import all_entries, latest_activity_at
from deerflow.skills.types import SkillCategory
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
agent_id = _resolve_active_agent_id(runtime)
allowed = _load_agent_skill_whitelist(agent_id)
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
# 当前用户"的技能。owned 为 None 表示 SkillStore 不可用,降级放行全部自定义
# 技能(见 _load_owned_skill_names)。
owned: set[str] | None = None
if allowed is None:
owned = _load_owned_skill_names(_resolve_effective_user_id())
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=False)
usage = all_entries()
result = []
for skill in skills:
if allowed is not None and skill.name not in allowed:
continue
entry = usage.get(skill.name, {})
if allowed is not None:
# 自定义 agent:严格按 config.yaml 的 skills 白名单过滤。
if skill.name not in allowed:
continue
else:
# 默认 agent:跳过已归档技能。
if entry.get("state") == STATE_ARCHIVED:
continue
# 自定义技能仅当属于当前用户时可见;内置 public 技能始终可见。
if skill.category == SkillCategory.CUSTOM and owned is not None and skill.name not in owned:
continue
result.append({
"name": skill.name,
"description": skill.description,
@@ -84,12 +84,26 @@ def get_available_tools(
builtin_tools = BUILTIN_TOOLS.copy()
if not is_scheduled_run:
builtin_tools.append(scheduled_task_tool)
skill_evolution_config = getattr(config, "skill_evolution", None)
# skill_list / skill_view 是只读工具(发现、查看技能),与"自我进化"写能力无关,
# 始终提供给 agent。
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
builtin_tools.extend([skill_list_tool, skill_view_tool])
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
try:
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
from deerflow.runtime.user_context import get_effective_user_id
skill_evolution_config = load_skill_evolution_config_for_user(get_effective_user_id())
except Exception: # noqa: BLE001 - 解析失败不能阻断 agent 构建
skill_evolution_config = getattr(config, "skill_evolution", None)
if getattr(skill_evolution_config, "enabled", False):
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
from deerflow.tools.skill_manage_tool import skill_manage_tool
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
builtin_tools.append(skill_manage_tool)
# Add subagent tools only if enabled via runtime parameter
if subagent_enabled: