Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into dhk-1
This commit is contained in:
commit
e08e7be24c
404 files changed
+113007
-921
No files matched your search
+43
-27
@@ -8,9 +8,10 @@ from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.memory.summarization_hook import memory_flush_hook
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
|
||||
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
|
||||
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
|
||||
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
|
||||
@@ -257,6 +258,10 @@ def _build_middlewares(
|
||||
resolved_app_config = app_config or get_app_config()
|
||||
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
|
||||
|
||||
# Inject the admin prompt prefix into the model request (kept out of the
|
||||
# persisted/displayed message so the user never sees it).
|
||||
middlewares.append(PromptPrefixMiddleware())
|
||||
|
||||
# Add summarization middleware if enabled
|
||||
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
|
||||
if summarization_middleware is not None:
|
||||
@@ -304,10 +309,10 @@ def _build_middlewares(
|
||||
if custom_middlewares:
|
||||
middlewares.extend(custom_middlewares)
|
||||
|
||||
middlewares.append(SetupAgentApprovalMiddleware())
|
||||
|
||||
# ClarificationMiddleware should always be last
|
||||
middlewares.append(ClarificationMiddleware())
|
||||
middlewares.append(SetupAgentApprovalMiddleware())
|
||||
|
||||
# ClarificationMiddleware should always be last
|
||||
middlewares.append(ClarificationMiddleware())
|
||||
return middlewares
|
||||
|
||||
|
||||
@@ -331,9 +336,13 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
requested_model_name: str | None = cfg.get("model_name") or cfg.get("model")
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
subagent_enabled = cfg.get("subagent_enabled", False)
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
writing_mode = bool(cfg.get("writing_mode", False))
|
||||
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
|
||||
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
|
||||
writing_artifact_path = None
|
||||
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
|
||||
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
agent_display_name = agent_config.name if agent_config else agent_id
|
||||
@@ -352,14 +361,15 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
thinking_enabled = False
|
||||
|
||||
logger.info(
|
||||
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s",
|
||||
agent_id or "default",
|
||||
thinking_enabled,
|
||||
reasoning_effort,
|
||||
model_name,
|
||||
is_plan_mode,
|
||||
subagent_enabled,
|
||||
max_concurrent_subagents,
|
||||
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s",
|
||||
agent_id or "default",
|
||||
thinking_enabled,
|
||||
reasoning_effort,
|
||||
model_name,
|
||||
is_plan_mode,
|
||||
subagent_enabled,
|
||||
max_concurrent_subagents,
|
||||
writing_mode,
|
||||
)
|
||||
|
||||
# Inject run metadata for LangSmith trace tagging
|
||||
@@ -383,8 +393,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
"thinking_enabled": thinking_enabled,
|
||||
"reasoning_effort": reasoning_effort,
|
||||
"is_plan_mode": is_plan_mode,
|
||||
"subagent_enabled": subagent_enabled,
|
||||
"tool_groups": agent_config.tool_groups if agent_config else None,
|
||||
"subagent_enabled": subagent_enabled,
|
||||
"writing_mode": writing_mode,
|
||||
"writing_artifact_path": writing_artifact_path,
|
||||
"tool_groups": agent_config.tool_groups if agent_config else None,
|
||||
"available_skills": runtime_available_skills,
|
||||
}
|
||||
)
|
||||
@@ -403,9 +415,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
middleware=_build_middlewares(config, model_name=model_name, app_config=resolved_app_config),
|
||||
system_prompt=apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
available_skills=set(["bootstrap"]),
|
||||
app_config=resolved_app_config,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
writing_mode=writing_mode,
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=set(["bootstrap"]),
|
||||
app_config=resolved_app_config,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
@@ -424,10 +438,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
system_prompt=apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_display_name,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_display_name,
|
||||
writing_mode=writing_mode,
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+50
-2
@@ -69,8 +69,25 @@ _enabled_skills_refresh_version = 0
|
||||
_enabled_skills_refresh_event = threading.Event()
|
||||
|
||||
|
||||
def _drop_archived_skills(skills: list[Skill]) -> list[Skill]:
|
||||
"""从注入提示词的技能列表里剔除已归档(usage state=archived)的技能。
|
||||
|
||||
技能归档是软状态:SKILL.md 仍留在 custom/<name>/、enabled 标志不变,所以
|
||||
load_skills(enabled_only=True) 仍会返回它。但归档技能不应再出现在 agent 的系统
|
||||
提示词里 —— 对 agent 而言它应等同于"不存在"。失败时不过滤(返回原列表)。
|
||||
"""
|
||||
try:
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
|
||||
|
||||
entries = all_entries()
|
||||
return [s for s in skills if entries.get(s.name, {}).get("state") != STATE_ARCHIVED]
|
||||
except Exception:
|
||||
logger.debug("Failed to filter archived skills for prompt injection", exc_info=True)
|
||||
return skills
|
||||
|
||||
|
||||
def _load_enabled_skills_sync() -> list[Skill]:
|
||||
return list(get_or_new_skill_storage().load_skills(enabled_only=True))
|
||||
return _drop_archived_skills(list(get_or_new_skill_storage().load_skills(enabled_only=True)))
|
||||
|
||||
|
||||
def _start_enabled_skills_refresh_thread() -> None:
|
||||
@@ -171,7 +188,7 @@ def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[
|
||||
"""
|
||||
if app_config is None:
|
||||
return _get_enabled_skills()
|
||||
return list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True))
|
||||
return _drop_archived_skills(list(get_or_new_skill_storage(app_config=app_config).load_skills(enabled_only=True)))
|
||||
|
||||
|
||||
def _skill_mutability_label(category: SkillCategory | str) -> str:
|
||||
@@ -481,6 +498,8 @@ You: "Deploying to staging..." [proceed]
|
||||
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
|
||||
{subagent_section}
|
||||
|
||||
<working_directory existed="true">
|
||||
@@ -800,6 +819,8 @@ def apply_prompt_template(
|
||||
*,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> str:
|
||||
@@ -835,6 +856,32 @@ def apply_prompt_template(
|
||||
|
||||
# Get deferred tools section (tool_search)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
current_writing_artifact = (
|
||||
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
|
||||
if writing_artifact_path
|
||||
else ""
|
||||
)
|
||||
writing_mode_section = (
|
||||
f"""
|
||||
<writing_mode>
|
||||
Writing mode is enabled for this run.
|
||||
|
||||
**Mandatory Output Contract**
|
||||
- Treat the user's request as a writing/report-generation task.
|
||||
- Your final deliverable MUST be a Markdown file, not only an inline chat answer.
|
||||
- If a current Markdown artifact is provided below, you MUST revise that existing file rather than creating a new file, unless the user explicitly asks for a separate/new document.
|
||||
- When revising an existing Markdown artifact, read the file first if needed, then overwrite the same path with the complete updated Markdown content.
|
||||
- Write the deliverable to `/mnt/user-data/outputs/` with a clear `.md` filename, for example `/mnt/user-data/outputs/report.md`.
|
||||
- Use `write_file` or an equivalent file-writing tool to save the complete Markdown content.
|
||||
- After writing the file, call `present_files` with the Markdown file path so the user can open it as an artifact/canvas.
|
||||
- The final chat response should be brief and point to the generated Markdown artifact. Do not duplicate the full report inline unless the user explicitly asks.
|
||||
- If you revise an existing Markdown artifact, update the Markdown file and present it again.
|
||||
{current_writing_artifact}
|
||||
</writing_mode>
|
||||
"""
|
||||
if writing_mode
|
||||
else ""
|
||||
)
|
||||
|
||||
# Build ACP agent section only if ACP agents are configured
|
||||
acp_section = _build_acp_section(app_config=app_config)
|
||||
@@ -850,6 +897,7 @@ def apply_prompt_template(
|
||||
soul=get_agent_soul(storage_agent_id),
|
||||
skills_section=skills_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
memory_context=memory_context,
|
||||
network_policy_section=network_policy_section,
|
||||
subagent_section=subagent_section,
|
||||
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
"""Middleware that records per-LLM-call metrics for the admin audit page.
|
||||
|
||||
Captures **every** provider invocation regardless of the frontend speed-meter
|
||||
toggle: the goal is a complete admin-side audit log of LLM usage, including
|
||||
failures. The user-facing speed indicator (controlled by
|
||||
``LocalSettings.display.streamSpeedMeterEnabled``) is a separate, purely
|
||||
cosmetic feature — turning it off does not stop this recording.
|
||||
|
||||
Each call ends with one row in ``llm_call_metrics``:
|
||||
- Successes record duration, token counts, and ``tokens_per_sec``.
|
||||
- Errors record the duration spent before the failure, the exception class as
|
||||
``error_type`` and a one-line ``error_message``.
|
||||
|
||||
Writes go through a best-effort store: a DB outage logs a warning and the
|
||||
agent runtime keeps streaming. We intentionally place this middleware
|
||||
*inside* :class:`LLMErrorHandlingMiddleware` (i.e. nested deeper in the
|
||||
``wrap_model_call`` chain) so we observe the **real** provider call timings —
|
||||
not the retry-wrapped successful response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import (
|
||||
ModelCallResult,
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
)
|
||||
from langchain_core.messages import AIMessage
|
||||
from langgraph.errors import GraphBubbleUp
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.llm_metrics import LlmMetricRecord, make_llm_metrics_store
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _safe_get_thread_id(request: ModelRequest) -> str | None:
|
||||
state = getattr(request, "state", None)
|
||||
if isinstance(state, dict):
|
||||
thread_id = state.get("thread_id")
|
||||
if isinstance(thread_id, str) and thread_id:
|
||||
return thread_id
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
config = getattr(runtime, "config", None)
|
||||
configurable = (config or {}).get("configurable", {}) if isinstance(config, dict) else {}
|
||||
thread_id = configurable.get("thread_id") if isinstance(configurable, dict) else None
|
||||
if isinstance(thread_id, str) and thread_id:
|
||||
return thread_id
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_run_id(request: ModelRequest) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
config = getattr(runtime, "config", None)
|
||||
if isinstance(config, dict):
|
||||
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
|
||||
if isinstance(metadata, dict):
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = metadata.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_model_name(request: ModelRequest) -> str | None:
|
||||
model = getattr(request, "model", None)
|
||||
for attr in ("model_name", "model", "name"):
|
||||
value = getattr(model, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_agent_name(request: ModelRequest) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in ("agent_name", "agent_id"):
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_usage(response: Any) -> dict[str, int]:
|
||||
"""Pull token counts out of an AIMessage, falling back to provider-native shapes.
|
||||
|
||||
The handler in ``wrap_model_call`` actually hands us a
|
||||
:class:`langchain.agents.middleware.types.ModelResponse` dataclass whose
|
||||
``.result`` is a ``list[BaseMessage]``. Earlier versions of this helper
|
||||
only matched ``AIMessage`` and ``list``, which meant we silently dropped
|
||||
every successful call and stored None for input/output/total. To stay
|
||||
robust we now also handle ``ModelResponse`` (and ``ExtendedModelResponse``,
|
||||
which wraps a ModelResponse), and we walk the messages list for the first
|
||||
``AIMessage``.
|
||||
|
||||
LangChain normalises usage into ``message.usage_metadata`` with the keys
|
||||
``input_tokens`` / ``output_tokens`` / ``total_tokens``. Several
|
||||
OpenAI-compatible providers (DeepSeek, vLLM, some Azure deployments) leave
|
||||
that dict partial — typically only ``total_tokens`` is populated — and the
|
||||
real breakdown sits under ``response_metadata.token_usage`` with the
|
||||
provider-native key names ``prompt_tokens`` / ``completion_tokens``. We
|
||||
merge both so the admin audit view always has the most complete numbers
|
||||
available.
|
||||
"""
|
||||
message: AIMessage | None = None
|
||||
|
||||
def _find_ai(candidate: Any) -> AIMessage | None:
|
||||
if isinstance(candidate, AIMessage):
|
||||
return candidate
|
||||
if isinstance(candidate, list):
|
||||
for item in candidate:
|
||||
found = _find_ai(item)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
message = _find_ai(response)
|
||||
if message is None:
|
||||
# ModelResponse / ExtendedModelResponse — defined as dataclasses with a
|
||||
# ``result`` field holding ``list[BaseMessage]``. Recurse via attribute
|
||||
# access so we don't tightly couple to the dataclass class itself.
|
||||
for attr in ("result", "model_response"):
|
||||
inner = getattr(response, attr, None)
|
||||
if inner is None:
|
||||
continue
|
||||
message = _find_ai(inner)
|
||||
if message is None and attr == "model_response":
|
||||
inner_result = getattr(inner, "result", None)
|
||||
message = _find_ai(inner_result)
|
||||
if message is not None:
|
||||
break
|
||||
if message is None:
|
||||
return {}
|
||||
|
||||
out: dict[str, int] = {}
|
||||
|
||||
usage = getattr(message, "usage_metadata", None) or {}
|
||||
if isinstance(usage, dict):
|
||||
for key in ("input_tokens", "output_tokens", "total_tokens"):
|
||||
coerced = _coerce_int(usage.get(key))
|
||||
if coerced is not None:
|
||||
out[key] = coerced
|
||||
|
||||
response_metadata = getattr(message, "response_metadata", None) or {}
|
||||
token_usage = response_metadata.get("token_usage") if isinstance(response_metadata, dict) else None
|
||||
if isinstance(token_usage, dict):
|
||||
# Map both the LangChain-canonical names and OpenAI-native names; later
|
||||
# writes win so a more-precise provider-native value can override a
|
||||
# zero placeholder from usage_metadata.
|
||||
candidates = {
|
||||
"input_tokens": ("input_tokens", "prompt_tokens"),
|
||||
"output_tokens": ("output_tokens", "completion_tokens"),
|
||||
"total_tokens": ("total_tokens",),
|
||||
}
|
||||
for canonical, keys in candidates.items():
|
||||
for key in keys:
|
||||
coerced = _coerce_int(token_usage.get(key))
|
||||
if coerced is not None and coerced > 0:
|
||||
out[canonical] = coerced
|
||||
break
|
||||
|
||||
# Derive a missing or implausibly-zero field if the other two are present.
|
||||
# Some providers report ``input_tokens=0`` / ``output_tokens=0`` next to a
|
||||
# real ``total_tokens`` — treat those zeros as missing so we don't silently
|
||||
# store nonsense breakdowns.
|
||||
def _missing_or_zero(key: str) -> bool:
|
||||
return out.get(key) in (None, 0)
|
||||
|
||||
if "total_tokens" not in out and out.get("input_tokens") and out.get("output_tokens"):
|
||||
out["total_tokens"] = out["input_tokens"] + out["output_tokens"]
|
||||
if _missing_or_zero("output_tokens") and out.get("total_tokens") and out.get("input_tokens"):
|
||||
derived = out["total_tokens"] - out["input_tokens"]
|
||||
if derived > 0:
|
||||
out["output_tokens"] = derived
|
||||
if _missing_or_zero("input_tokens") and out.get("total_tokens") and out.get("output_tokens"):
|
||||
derived = out["total_tokens"] - out["output_tokens"]
|
||||
if derived > 0:
|
||||
out["input_tokens"] = derived
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _classify_exception(exc: BaseException) -> tuple[str, str]:
|
||||
"""Return (error_type, error_message). Message is trimmed to 500 chars."""
|
||||
name = exc.__class__.__name__
|
||||
raw = (str(exc) or name).strip()
|
||||
return name, (raw[:500] if len(raw) > 500 else raw)
|
||||
|
||||
|
||||
class LlmMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Wrap each model call to time it and persist a metric row."""
|
||||
|
||||
def __init__(self, *, store: LlmMetricsStore | None = None, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._store_override = store
|
||||
|
||||
def _store(self) -> LlmMetricsStore | None:
|
||||
if self._store_override is not None:
|
||||
return self._store_override
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
except Exception:
|
||||
return None
|
||||
return make_llm_metrics_store(sf)
|
||||
|
||||
def _build_base_record(self, request: ModelRequest) -> LlmMetricRecord:
|
||||
return LlmMetricRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=_now(),
|
||||
user_id=get_effective_user_id(),
|
||||
thread_id=_safe_get_thread_id(request),
|
||||
run_id=_safe_get_run_id(request),
|
||||
agent_name=_safe_get_agent_name(request),
|
||||
model_name=_safe_get_model_name(request),
|
||||
)
|
||||
|
||||
def _finalize_success(self, record: LlmMetricRecord, response: ModelResponse, started: float) -> None:
|
||||
elapsed = max(0.0, time.monotonic() - started)
|
||||
record.duration_ms = int(elapsed * 1000)
|
||||
usage = _extract_usage(response)
|
||||
record.input_tokens = usage.get("input_tokens")
|
||||
record.output_tokens = usage.get("output_tokens")
|
||||
record.total_tokens = usage.get("total_tokens") or (
|
||||
(record.input_tokens or 0) + (record.output_tokens or 0) or None
|
||||
)
|
||||
if record.output_tokens is not None and elapsed > 0:
|
||||
record.tokens_per_sec = round(record.output_tokens / elapsed, 3)
|
||||
record.status = "success"
|
||||
|
||||
def _finalize_error(self, record: LlmMetricRecord, exc: BaseException, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status = "error"
|
||||
record.error_type, record.error_message = _classify_exception(exc)
|
||||
|
||||
def _flush(self, record: LlmMetricRecord) -> None:
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the request path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("LlmMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = handler(request)
|
||||
except GraphBubbleUp:
|
||||
# LangGraph control flow — not a real LLM failure.
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_success(record, response, started)
|
||||
self._flush(record)
|
||||
return response
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = await handler(request)
|
||||
except GraphBubbleUp:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_success(record, response, started)
|
||||
self._flush(record)
|
||||
return response
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
"""Middleware that injects the admin-configured prompt prefix at model-call time.
|
||||
|
||||
The prompt-prefix feature lets an admin prepend a fixed instruction to the
|
||||
user's question before it reaches the LLM. To keep the prefix invisible in the
|
||||
chat UI, the Gateway does **not** concatenate it into the persisted
|
||||
``HumanMessage.content``; instead it tags the first human message with
|
||||
``additional_kwargs["prompt_prefix"]`` (see ``app.gateway.services``).
|
||||
|
||||
This middleware reads that tag and prepends the prefix to the message content
|
||||
*only in the request handed to the model*. The persisted state — and therefore
|
||||
what the frontend renders — keeps the clean, prefix-free content.
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
PROMPT_PREFIX_KEY = "prompt_prefix"
|
||||
|
||||
|
||||
class PromptPrefixMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Prepends ``additional_kwargs['prompt_prefix']`` to human messages for the LLM only.
|
||||
|
||||
The transformation is applied to the model request copy, never written back
|
||||
to state, so it is naturally idempotent across the multiple model calls of a
|
||||
single turn.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _apply(messages: list) -> list | None:
|
||||
"""Return a patched copy with prefixes prepended, or None if nothing to do."""
|
||||
patched: list | None = None
|
||||
for idx, msg in enumerate(messages):
|
||||
if not isinstance(msg, HumanMessage):
|
||||
continue
|
||||
raw_prefix = (getattr(msg, "additional_kwargs", None) or {}).get(PROMPT_PREFIX_KEY)
|
||||
prefix = str(raw_prefix).strip() if raw_prefix else ""
|
||||
if not prefix:
|
||||
continue
|
||||
content = msg.content
|
||||
if isinstance(content, str):
|
||||
new_content: object = f"{prefix}\n\n{content}".strip("\n")
|
||||
elif isinstance(content, list):
|
||||
# Structured content (text blocks + media): prepend a text block.
|
||||
new_content = [{"type": "text", "text": prefix}, *content]
|
||||
else:
|
||||
new_content = f"{prefix}\n\n{content!s}"
|
||||
if patched is None:
|
||||
patched = list(messages)
|
||||
# model_copy preserves the message id so streaming dedup is unaffected.
|
||||
patched[idx] = msg.model_copy(update={"content": new_content})
|
||||
return patched
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
patched = self._apply(request.messages)
|
||||
if patched is not None:
|
||||
request = request.override(messages=patched)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
patched = self._apply(request.messages)
|
||||
if patched is not None:
|
||||
request = request.override(messages=patched)
|
||||
return await handler(request)
|
||||
+1
-1
@@ -72,7 +72,7 @@ class SetupAgentApprovalMiddleware(AgentMiddleware[SetupAgentApprovalMiddlewareS
|
||||
|
||||
approval_msg = ToolMessage(
|
||||
id=self._stable_message_id(tool_call_id, args),
|
||||
content="请确认是否创建/更新这个专家配置。",
|
||||
content="请确认是否创建/更新这个智能体配置。",
|
||||
tool_call_id=tool_call_id,
|
||||
name="setup_agent",
|
||||
additional_kwargs={"setup_agent_approval": payload},
|
||||
|
||||
+8
@@ -96,6 +96,14 @@ def _build_runtime_middlewares(
|
||||
|
||||
middlewares.append(LLMErrorHandlingMiddleware(app_config=app_config))
|
||||
|
||||
# LLM call metrics — always on, even when the user-facing speed indicator
|
||||
# is disabled. We append *after* the retry middleware so the metrics
|
||||
# middleware sits deeper in the wrap chain and observes individual provider
|
||||
# attempts (including the failed ones).
|
||||
from deerflow.agents.middlewares.llm_metrics_middleware import LlmMetricsMiddleware
|
||||
|
||||
middlewares.append(LlmMetricsMiddleware())
|
||||
|
||||
# Guardrail middleware (if configured)
|
||||
guardrails_config = app_config.guardrails
|
||||
if guardrails_config.enabled and guardrails_config.provider:
|
||||
|
||||
+6
-1
@@ -45,7 +45,12 @@ class McpServerConfig(BaseModel):
|
||||
headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers to send (for sse or http type)")
|
||||
oauth: McpOAuthConfig | None = Field(default=None, description="OAuth configuration (for sse or http type)")
|
||||
description: str = Field(default="", description="Human-readable description of what this MCP server provides")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
disabled_tools: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="List of tool names (without server-name prefix) to hide from the agent",
|
||||
alias="disabledTools",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""System-wide runtime settings editable from the admin UI.
|
||||
|
||||
Backed by a single JSON file under the DeerFlow runtime home (default
|
||||
``.deer-flow/system_settings.json``). Unlike :mod:`extensions_config` this
|
||||
file is **not** part of the project source tree — it is per-deployment
|
||||
state, and it stays writable at runtime.
|
||||
|
||||
The only feature stored here at the moment is the **prompt prefix toggle**
|
||||
shown in the chat input: an administrator can configure a label and a
|
||||
prefix string, and any user can flip the toggle on/off; when enabled, the
|
||||
prefix is prepended to the first human message of a new thread.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SETTINGS_FILE = "system_settings.json"
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
class PromptPrefixSettings(BaseModel):
|
||||
"""Admin-configurable first-message prompt prefix."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the prefix toggle is offered to users at all")
|
||||
switch_label: str = Field(default="加入立场前缀", description="Toggle label shown in the chat input")
|
||||
prompt_prefix: str = Field(default="", description="Text prepended to the first user message when enabled")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SystemSettings(BaseModel):
|
||||
"""Top-level system settings document."""
|
||||
|
||||
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
def _settings_path() -> Path:
|
||||
home = runtime_home()
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
return home / _SETTINGS_FILE
|
||||
|
||||
|
||||
def load_system_settings() -> SystemSettings:
|
||||
"""Read the on-disk system settings, returning defaults when missing."""
|
||||
path = _settings_path()
|
||||
if not path.is_file():
|
||||
return SystemSettings()
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
logger.warning("System settings file %s is not a JSON object; ignoring", path)
|
||||
return SystemSettings()
|
||||
return SystemSettings.model_validate(raw)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read system settings from %s: %s", path, exc)
|
||||
return SystemSettings()
|
||||
|
||||
|
||||
def save_system_settings(settings: SystemSettings) -> SystemSettings:
|
||||
"""Persist the settings document atomically, returning the saved value."""
|
||||
path = _settings_path()
|
||||
payload = settings.model_dump(mode="json")
|
||||
with _lock:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
return settings
|
||||
@@ -121,6 +121,25 @@ async def get_mcp_tools() -> list[BaseTool]:
|
||||
|
||||
# Get all tools from all servers
|
||||
tools = await client.get_tools()
|
||||
|
||||
# Filter out tools listed in each server's `disabled_tools`. Tool names from
|
||||
# MultiServerMCPClient are prefixed with the server name (because we pass
|
||||
# `tool_name_prefix=True`), so we compare against `<server>_<tool>` here.
|
||||
disabled_full_names: set[str] = set()
|
||||
for server_name, server_cfg in extensions_config.get_enabled_mcp_servers().items():
|
||||
for tool_name in server_cfg.disabled_tools or []:
|
||||
name = (tool_name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
disabled_full_names.add(f"{server_name}_{name}")
|
||||
disabled_full_names.add(name)
|
||||
if disabled_full_names:
|
||||
kept = [t for t in tools if t.name not in disabled_full_names]
|
||||
removed = len(tools) - len(kept)
|
||||
if removed:
|
||||
logger.info(f"Filtered out {removed} disabled MCP tool(s)")
|
||||
tools = kept
|
||||
|
||||
logger.info(f"Successfully loaded {len(tools)} tool(s) from MCP servers")
|
||||
|
||||
# Patch tools to support sync invocation, as deerflow client streams synchronously
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence layer for per-LLM-call metrics.
|
||||
|
||||
Each completed LLM invocation (success or failure) writes a single row so
|
||||
administrators can audit usage and speed. Designed to be cheap to write —
|
||||
the middleware fires once per provider call and never blocks streaming.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"LlmMetricRecord",
|
||||
"LlmMetricsStore",
|
||||
"SqlLlmMetricsStore",
|
||||
"make_llm_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlLlmMetricsStore(session_factory)
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Abstract repository contract for LLM call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricRecord:
|
||||
"""In-memory shape of a metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
model_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
tokens_per_sec: float | None = None
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
model_name: str | None = None
|
||||
status: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class LlmMetricsStore(ABC):
|
||||
"""Persistence contract for the metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""ORM model for per-LLM-call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
"""One row per completed (or failed) LLM provider invocation."""
|
||||
|
||||
__tablename__ = "llm_call_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
# Wall-clock duration of the provider invocation in milliseconds. Always set.
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
input_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
output_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
total_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
# Output tokens / seconds; null when there is no completion (errors).
|
||||
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
# "success" or "error". Errors store the classification and original message
|
||||
# so admins can scan for transient vs auth/quota issues.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"model_name": self.model_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
"tokens_per_sec": self.tokens_per_sec,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""SQLAlchemy-backed LLM metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
row = LlmCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
model_name=metric.model_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
input_tokens=metric.input_tokens,
|
||||
output_tokens=metric.output_tokens,
|
||||
total_tokens=metric.total_tokens,
|
||||
tokens_per_sec=metric.tokens_per_sec,
|
||||
status=metric.status,
|
||||
error_type=metric.error_type,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort. Never let a persistence failure surface
|
||||
# to the agent runtime — log and move on.
|
||||
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
def _apply_filters(self, stmt, query: LlmMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(LlmCallMetricRow.status == query.status)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
# Page through the result so the Excel export stays bounded in memory
|
||||
# for large date windows.
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page_query = LlmMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
model_name=query.model_name,
|
||||
status=query.status,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
page = await self.list(page_query)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""Create llm_call_metrics table.
|
||||
|
||||
Revision ID: 20260514_02
|
||||
Revises: 20260514_01
|
||||
Create Date: 2026-05-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260514_02"
|
||||
down_revision = "20260514_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"llm_call_metrics",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=True),
|
||||
sa.Column("run_id", sa.String(64), nullable=True),
|
||||
sa.Column("agent_name", sa.String(128), nullable=True),
|
||||
sa.Column("model_name", sa.String(128), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("input_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("output_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("total_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("tokens_per_sec", sa.Float(), nullable=True),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
|
||||
sa.Column("error_type", sa.String(64), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_llm_call_metrics_created_at", "llm_call_metrics", ["created_at"])
|
||||
op.create_index("ix_llm_call_metrics_user_id", "llm_call_metrics", ["user_id"])
|
||||
op.create_index("ix_llm_call_metrics_model_name", "llm_call_metrics", ["model_name"])
|
||||
op.create_index("ix_llm_call_metrics_status", "llm_call_metrics", ["status"])
|
||||
op.create_index("ix_llm_call_metrics_thread_id", "llm_call_metrics", ["thread_id"])
|
||||
op.create_index("ix_llm_call_metrics_run_id", "llm_call_metrics", ["run_id"])
|
||||
op.create_index("ix_llm_call_metrics_agent_name", "llm_call_metrics", ["agent_name"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
op.drop_table("llm_call_metrics")
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
"""Create tags and tag_assignments tables.
|
||||
|
||||
Revision ID: 20260515_01
|
||||
Revises: 20260514_02
|
||||
Create Date: 2026-05-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260515_01"
|
||||
down_revision = "20260514_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("tags"):
|
||||
op.create_table(
|
||||
"tags",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("tag_type", sa.String(32), nullable=False),
|
||||
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),
|
||||
)
|
||||
op.create_index("ix_tags_tag_type", "tags", ["tag_type"])
|
||||
|
||||
if not _has_table("tag_assignments"):
|
||||
op.create_table(
|
||||
"tag_assignments",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("tag_id", sa.String(64), nullable=False),
|
||||
sa.Column("target_type", sa.String(32), nullable=False),
|
||||
sa.Column("target_id", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
|
||||
)
|
||||
op.create_index("ix_tag_assignments_tag_id", "tag_assignments", ["tag_id"])
|
||||
op.create_index("ix_tag_assignments_target_type", "tag_assignments", ["target_type"])
|
||||
op.create_index("ix_tag_assignments_target_id", "tag_assignments", ["target_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("tag_assignments"):
|
||||
op.drop_table("tag_assignments")
|
||||
if _has_table("tags"):
|
||||
op.drop_table("tags")
|
||||
+7
@@ -17,8 +17,10 @@ there is no matching entity directory.
|
||||
|
||||
from deerflow.persistence.agents.model import AgentRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
ScheduledTaskDeliveryProfileRow,
|
||||
@@ -28,13 +30,16 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
SchedulerThreadRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = [
|
||||
"FeedbackRow",
|
||||
"AgentRow",
|
||||
"LlmCallMetricRow",
|
||||
"NotificationRow",
|
||||
"RecommendedQuestionRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
@@ -43,6 +48,8 @@ __all__ = [
|
||||
"ScheduledTaskSubscriptionRow",
|
||||
"SchedulerThreadRow",
|
||||
"SkillRow",
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
"UserRow",
|
||||
]
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
"""Persistence layer for tags and tag-to-resource assignments."""
|
||||
|
||||
from deerflow.persistence.tags.base import TagStore
|
||||
from deerflow.persistence.tags.memory import MemoryTagStore
|
||||
from deerflow.persistence.tags.model import TAG_TYPES
|
||||
from deerflow.persistence.tags.sql import TagRepository
|
||||
|
||||
__all__ = ["TagStore", "MemoryTagStore", "TagRepository", "TAG_TYPES", "make_tag_store"]
|
||||
|
||||
|
||||
def make_tag_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryTagStore()
|
||||
return TagRepository(session_factory)
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Abstract persistence API for tags and tag-to-resource assignments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TagStore(ABC):
|
||||
# ------------------------------------------------------------------ tags
|
||||
|
||||
@abstractmethod
|
||||
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
|
||||
"""List tags, optionally filtered by ``tag_type``.
|
||||
|
||||
Each record includes a derived ``usage_count`` (number of assignments).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a tag. ``data`` must carry ``id``, ``name``, ``tag_type``.
|
||||
|
||||
Raises ``ValueError`` if ``(tag_type, name)`` is not unique.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update ``name`` / ``color`` / ``description``. ``tag_type`` is immutable.
|
||||
|
||||
Returns ``None`` if the tag does not exist. Raises ``ValueError`` on a
|
||||
name collision within the type.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_tag(self, tag_id: str) -> bool:
|
||||
"""Delete a tag and cascade-remove all of its assignments."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ----------------------------------------------------------- assignments
|
||||
|
||||
@abstractmethod
|
||||
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
|
||||
"""Return the tags currently assigned to one resource."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
|
||||
"""Replace the full set of tags on a resource.
|
||||
|
||||
Every id in ``tag_ids`` must reference an existing tag whose
|
||||
``tag_type`` equals ``target_type``; otherwise raises ``ValueError``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Bulk-fetch tags for many resources at once (avoids N+1 on list pages).
|
||||
|
||||
Returns ``{target_id: [tag, ...]}`` for ids that have at least one tag.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def unassign_all(self, target_type: str, target_id: str) -> int:
|
||||
"""Drop every tag assignment for a resource. Returns the count removed."""
|
||||
raise NotImplementedError
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
"""In-memory ``TagStore`` implementation for tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.tags.base import TagStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(record: dict[str, Any]) -> dict[str, Any]:
|
||||
return copy.deepcopy(record)
|
||||
|
||||
|
||||
class MemoryTagStore(TagStore):
|
||||
def __init__(self) -> None:
|
||||
self._tags: dict[str, dict[str, Any]] = {}
|
||||
# assignments keyed by surrogate id
|
||||
self._assignments: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# ------------------------------------------------------------------ tags
|
||||
|
||||
def _usage_count(self, tag_id: str) -> int:
|
||||
return sum(1 for a in self._assignments.values() if a["tag_id"] == tag_id)
|
||||
|
||||
def _with_count(self, record: dict[str, Any]) -> dict[str, Any]:
|
||||
out = _copy(record)
|
||||
out["usage_count"] = self._usage_count(record["id"])
|
||||
return out
|
||||
|
||||
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._tags.values() if tag_type is None or r["tag_type"] == tag_type]
|
||||
rows.sort(key=lambda r: r["created_at"])
|
||||
return [self._with_count(r) for r in rows]
|
||||
|
||||
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
|
||||
record = self._tags.get(tag_id)
|
||||
return self._with_count(record) if record is not None else None
|
||||
|
||||
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
tag_id = data["id"]
|
||||
name = data["name"]
|
||||
tag_type = data["tag_type"]
|
||||
if any(r["tag_type"] == tag_type and r["name"] == name for r in self._tags.values()):
|
||||
raise ValueError(f"Tag '{name}' already exists for type '{tag_type}'")
|
||||
record = {
|
||||
"id": tag_id,
|
||||
"name": name,
|
||||
"tag_type": tag_type,
|
||||
"color": data.get("color") or "gray",
|
||||
"description": data.get("description") or "",
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._tags[tag_id] = record
|
||||
return self._with_count(record)
|
||||
|
||||
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
record = self._tags.get(tag_id)
|
||||
if record is None:
|
||||
return None
|
||||
new_name = data.get("name", record["name"])
|
||||
if any(r["id"] != tag_id and r["tag_type"] == record["tag_type"] and r["name"] == new_name for r in self._tags.values()):
|
||||
raise ValueError(f"Tag '{new_name}' already exists for type '{record['tag_type']}'")
|
||||
if "name" in data:
|
||||
record["name"] = data["name"]
|
||||
if "color" in data:
|
||||
record["color"] = data["color"]
|
||||
if "description" in data:
|
||||
record["description"] = data["description"]
|
||||
record["updated_at"] = _now()
|
||||
return self._with_count(record)
|
||||
|
||||
async def delete_tag(self, tag_id: str) -> bool:
|
||||
if tag_id not in self._tags:
|
||||
return False
|
||||
del self._tags[tag_id]
|
||||
for aid in [aid for aid, a in self._assignments.items() if a["tag_id"] == tag_id]:
|
||||
del self._assignments[aid]
|
||||
return True
|
||||
|
||||
# ----------------------------------------------------------- assignments
|
||||
|
||||
def _tags_for(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
|
||||
tag_ids = [a["tag_id"] for a in self._assignments.values() if a["target_type"] == target_type and a["target_id"] == target_id]
|
||||
records = [self._tags[tid] for tid in tag_ids if tid in self._tags]
|
||||
records.sort(key=lambda r: r["name"])
|
||||
return [self._with_count(r) for r in records]
|
||||
|
||||
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
|
||||
return self._tags_for(target_type, target_id)
|
||||
|
||||
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
|
||||
unique_ids = list(dict.fromkeys(tag_ids))
|
||||
for tid in unique_ids:
|
||||
tag = self._tags.get(tid)
|
||||
if tag is None:
|
||||
raise ValueError(f"Tag '{tid}' does not exist")
|
||||
if tag["tag_type"] != target_type:
|
||||
raise ValueError(f"Tag '{tid}' is type '{tag['tag_type']}', cannot tag a '{target_type}' resource")
|
||||
# drop existing, then re-add
|
||||
for aid in [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]:
|
||||
del self._assignments[aid]
|
||||
for tid in unique_ids:
|
||||
aid = f"{tid}:{target_type}:{target_id}"
|
||||
self._assignments[aid] = {
|
||||
"id": aid,
|
||||
"tag_id": tid,
|
||||
"target_type": target_type,
|
||||
"target_id": target_id,
|
||||
"created_at": _now(),
|
||||
}
|
||||
return self._tags_for(target_type, target_id)
|
||||
|
||||
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
|
||||
wanted = set(target_ids)
|
||||
out: dict[str, list[dict[str, Any]]] = {}
|
||||
for a in self._assignments.values():
|
||||
if a["target_type"] != target_type or a["target_id"] not in wanted:
|
||||
continue
|
||||
tag = self._tags.get(a["tag_id"])
|
||||
if tag is None:
|
||||
continue
|
||||
out.setdefault(a["target_id"], []).append(self._with_count(tag))
|
||||
for tags in out.values():
|
||||
tags.sort(key=lambda r: r["name"])
|
||||
return out
|
||||
|
||||
async def unassign_all(self, target_type: str, target_id: str) -> int:
|
||||
victims = [aid for aid, a in self._assignments.items() if a["target_type"] == target_type and a["target_id"] == target_id]
|
||||
for aid in victims:
|
||||
del self._assignments[aid]
|
||||
return len(victims)
|
||||
@@ -0,0 +1,47 @@
|
||||
"""ORM models for tags and tag-to-resource assignments.
|
||||
|
||||
Two tables:
|
||||
|
||||
- ``tags`` — a tag belongs to exactly one ``tag_type`` (``agent`` / ``skill`` /
|
||||
``scheduled_task``). Tag names are unique *within* a type.
|
||||
- ``tag_assignments`` — many-to-many link between a tag and a target resource.
|
||||
``target_type`` is denormalized from the tag so the table is queryable by
|
||||
``(target_type, target_id)`` without a join. The API layer enforces the
|
||||
invariant ``target_type == tag.tag_type``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
# Allowed tag/target types. Kept here so persistence + API share one source.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
|
||||
|
||||
|
||||
class TagRow(Base):
|
||||
__tablename__ = "tags"
|
||||
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
__tablename__ = "tag_assignments"
|
||||
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
@@ -0,0 +1,187 @@
|
||||
"""SQLAlchemy-backed tag + tag-assignment repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tags.base import TagStore
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
|
||||
|
||||
def _tag_to_dict(row: TagRow, usage_count: int = 0) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
data["usage_count"] = usage_count
|
||||
return data
|
||||
|
||||
|
||||
class TagRepository(TagStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
async def _usage_counts(session: AsyncSession, tag_ids: list[str]) -> dict[str, int]:
|
||||
if not tag_ids:
|
||||
return {}
|
||||
stmt = select(TagAssignmentRow.tag_id, func.count()).where(TagAssignmentRow.tag_id.in_(tag_ids)).group_by(TagAssignmentRow.tag_id)
|
||||
result = await session.execute(stmt)
|
||||
return {tag_id: count for tag_id, count in result.all()}
|
||||
|
||||
# ------------------------------------------------------------------ tags
|
||||
|
||||
async def list_tags(self, tag_type: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(TagRow)
|
||||
if tag_type is not None:
|
||||
stmt = stmt.where(TagRow.tag_type == tag_type)
|
||||
stmt = stmt.order_by(TagRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
counts = await self._usage_counts(session, [r.id for r in rows])
|
||||
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
|
||||
|
||||
async def get_tag(self, tag_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TagRow, tag_id)
|
||||
if row is None:
|
||||
return None
|
||||
counts = await self._usage_counts(session, [tag_id])
|
||||
return _tag_to_dict(row, counts.get(tag_id, 0))
|
||||
|
||||
async def create_tag(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = TagRow(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
tag_type=data["tag_type"],
|
||||
color=data.get("color") or "gray",
|
||||
description=data.get("description") or "",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError as exc:
|
||||
await session.rollback()
|
||||
raise ValueError(f"Tag '{data['name']}' already exists for type '{data['tag_type']}'") from exc
|
||||
await session.refresh(row)
|
||||
return _tag_to_dict(row, 0)
|
||||
|
||||
async def update_tag(self, tag_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TagRow, tag_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data:
|
||||
row.name = data["name"]
|
||||
if "color" in data:
|
||||
row.color = data["color"]
|
||||
if "description" in data:
|
||||
row.description = data["description"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError as exc:
|
||||
await session.rollback()
|
||||
raise ValueError(f"Tag '{data.get('name')}' already exists for type '{row.tag_type}'") from exc
|
||||
await session.refresh(row)
|
||||
counts = await self._usage_counts(session, [tag_id])
|
||||
return _tag_to_dict(row, counts.get(tag_id, 0))
|
||||
|
||||
async def delete_tag(self, tag_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TagRow, tag_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.execute(delete(TagAssignmentRow).where(TagAssignmentRow.tag_id == tag_id))
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# ----------------------------------------------------------- assignments
|
||||
|
||||
async def _tags_for(self, session: AsyncSession, target_type: str, target_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(TagRow)
|
||||
.join(TagAssignmentRow, TagAssignmentRow.tag_id == TagRow.id)
|
||||
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id == target_id)
|
||||
.order_by(TagRow.name.asc())
|
||||
)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
counts = await self._usage_counts(session, [r.id for r in rows])
|
||||
return [_tag_to_dict(r, counts.get(r.id, 0)) for r in rows]
|
||||
|
||||
async def get_target_tags(self, target_type: str, target_id: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
return await self._tags_for(session, target_type, target_id)
|
||||
|
||||
async def set_target_tags(self, target_type: str, target_id: str, tag_ids: list[str]) -> list[dict[str, Any]]:
|
||||
unique_ids = list(dict.fromkeys(tag_ids))
|
||||
async with self._sf() as session:
|
||||
if unique_ids:
|
||||
found = list((await session.execute(select(TagRow).where(TagRow.id.in_(unique_ids)))).scalars())
|
||||
by_id = {r.id: r for r in found}
|
||||
for tid in unique_ids:
|
||||
tag = by_id.get(tid)
|
||||
if tag is None:
|
||||
raise ValueError(f"Tag '{tid}' does not exist")
|
||||
if tag.tag_type != target_type:
|
||||
raise ValueError(f"Tag '{tid}' is type '{tag.tag_type}', cannot tag a '{target_type}' resource")
|
||||
await session.execute(
|
||||
delete(TagAssignmentRow).where(
|
||||
TagAssignmentRow.target_type == target_type,
|
||||
TagAssignmentRow.target_id == target_id,
|
||||
)
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
for tid in unique_ids:
|
||||
session.add(
|
||||
TagAssignmentRow(
|
||||
id=uuid.uuid4().hex,
|
||||
tag_id=tid,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return await self._tags_for(session, target_type, target_id)
|
||||
|
||||
async def list_assignments(self, target_type: str, target_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
|
||||
if not target_ids:
|
||||
return {}
|
||||
stmt = (
|
||||
select(TagAssignmentRow.target_id, TagRow)
|
||||
.join(TagRow, TagAssignmentRow.tag_id == TagRow.id)
|
||||
.where(TagAssignmentRow.target_type == target_type, TagAssignmentRow.target_id.in_(target_ids))
|
||||
.order_by(TagRow.name.asc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
tag_rows = [tag for _, tag in rows]
|
||||
counts = await self._usage_counts(session, [t.id for t in tag_rows])
|
||||
out: dict[str, list[dict[str, Any]]] = {}
|
||||
for target_id, tag in rows:
|
||||
out.setdefault(target_id, []).append(_tag_to_dict(tag, counts.get(tag.id, 0)))
|
||||
return out
|
||||
|
||||
async def unassign_all(self, target_type: str, target_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(TagAssignmentRow).where(
|
||||
TagAssignmentRow.target_type == target_type,
|
||||
TagAssignmentRow.target_id == target_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount or 0
|
||||
+10
-2
@@ -247,7 +247,11 @@ class ScheduledTaskService:
|
||||
accessible = await self._accessible_task(user_id, task_id)
|
||||
if accessible is None:
|
||||
return []
|
||||
return await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
|
||||
runs = await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
|
||||
can_edit = accessible.get("role") == "owner"
|
||||
for run in runs:
|
||||
run["can_edit"] = can_edit
|
||||
return runs
|
||||
|
||||
async def get_run(self, user_id: str, run_id: str) -> dict[str, Any] | None:
|
||||
run = await self.store.get_run(run_id, user_id)
|
||||
@@ -316,7 +320,7 @@ class ScheduledTaskService:
|
||||
except ValueError as exc:
|
||||
raise PermissionError("Refusing to edit file outside the run archive") from exc
|
||||
|
||||
archive_path.write_text(new_content, encoding="utf-8")
|
||||
archive_path.write_text(new_content, encoding="utf-8", newline="")
|
||||
new_size = archive_path.stat().st_size
|
||||
|
||||
# Reflect the new size in the run row so subsequent list/preview calls
|
||||
@@ -342,6 +346,10 @@ class ScheduledTaskService:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
return True
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
|
||||
raise PermissionError("Only the run owner may delete scheduled task runs")
|
||||
return deleted
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
|
||||
@@ -264,7 +264,7 @@ def _eligible_for_curation(
|
||||
def should_run_now(user_id: str) -> bool:
|
||||
"""Return True if the curator interval has elapsed for ``user_id`` and it
|
||||
is not paused."""
|
||||
curator_cfg = _get_curator_config()
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
|
||||
return False
|
||||
state = _read_curator_state(user_id)
|
||||
@@ -278,7 +278,7 @@ def should_run_now(user_id: str) -> bool:
|
||||
last_dt = datetime.fromisoformat(last_run)
|
||||
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
|
||||
except Exception:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
|
||||
@@ -297,7 +297,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
|
||||
if now is None:
|
||||
now = datetime.now(UTC)
|
||||
|
||||
curator_cfg = _get_curator_config()
|
||||
curator_cfg = _get_curator_config(user_id)
|
||||
stale_days = getattr(curator_cfg, "stale_after_days", 30) if curator_cfg else 30
|
||||
archive_days = getattr(curator_cfg, "archive_after_days", 90) if curator_cfg else 90
|
||||
|
||||
@@ -590,8 +590,72 @@ def _make_skill_archive_tool(user_id: str):
|
||||
return _skill_archive
|
||||
|
||||
|
||||
def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
"""Run the curator consolidation pass as a React agent with skill tools bound."""
|
||||
def _make_curator_skill_manage_tool():
|
||||
"""curator 受限版 ``skill_manage``:在工具层屏蔽 delete 动作。
|
||||
|
||||
curator 做 umbrella 合并需要 create / patch / edit / write_file,但绝不能删整个
|
||||
技能 —— prompt 硬规则也写明"归档(skill_archive)是允许的最大操作"。通用
|
||||
skill_manage 的 delete 走 ``_assert_agent_may_delete``,**绕过了** skill_archive
|
||||
对"已发布 / 被 agent 引用"技能的拒绝保护。只靠 prompt 文字约束不可靠,这里在
|
||||
工具层直接拒掉 delete,curator 的 LLM 无论如何都删不掉技能。
|
||||
"""
|
||||
from langchain.tools import tool as lc_tool
|
||||
|
||||
from deerflow.tools.skill_manage_tool import _skill_manage_impl
|
||||
|
||||
@lc_tool("skill_manage")
|
||||
def _curator_skill_manage(
|
||||
action: str,
|
||||
name: str,
|
||||
content: str | None = None,
|
||||
path: str | None = None,
|
||||
find: str | None = None,
|
||||
replace: str | None = None,
|
||||
expected_count: int | None = None,
|
||||
) -> str:
|
||||
"""管理 skills/custom/ 下的自定义技能(curator 受限版)。
|
||||
|
||||
支持 action:create / patch / edit / write_file / remove_file / list / view。
|
||||
**不支持 delete** —— curator 不得删除技能;如需下架请改用 skill_archive。
|
||||
|
||||
Args:
|
||||
action: 操作类型(见上;delete 会被拒绝)。
|
||||
name: 连字符命名的技能名。
|
||||
content: create / edit / write_file 的新文件内容。
|
||||
path: write_file / remove_file 的支持文件路径。
|
||||
find: patch 要替换的原文本。
|
||||
replace: patch 的替换文本。
|
||||
expected_count: patch 的预期替换次数(可选)。
|
||||
"""
|
||||
if action == "delete":
|
||||
return (
|
||||
"Refused: curator 不允许删除技能。归档是允许的最大操作 —— "
|
||||
"请改用 skill_archive(name, reason) 软下架(文件保留,用户可在 Web UI 恢复)。"
|
||||
)
|
||||
# curator 不绑定到具体会话线程,runtime 传 None(_get_thread_id 兼容 None)。
|
||||
return _run_async_blocking(
|
||||
_skill_manage_impl(
|
||||
runtime=None,
|
||||
action=action,
|
||||
name=name,
|
||||
content=content,
|
||||
path=path,
|
||||
find=find,
|
||||
replace=replace,
|
||||
expected_count=expected_count,
|
||||
)
|
||||
)
|
||||
|
||||
return _curator_skill_manage
|
||||
|
||||
|
||||
def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dict]]:
|
||||
"""Run the curator consolidation pass as a React agent with skill tools bound.
|
||||
|
||||
返回 ``(llm_text, tool_actions)``:
|
||||
- ``llm_text`` —— LLM 最终回复文字(它"自述"做了什么);
|
||||
- ``tool_actions`` —— 从消息流提取的**真实**工具调用记录(它"实际"做了什么)。
|
||||
"""
|
||||
from deerflow.skills import curator_log
|
||||
|
||||
try:
|
||||
@@ -601,10 +665,10 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
from deerflow.tools.skill_manage_tool import skill_manage_tool
|
||||
|
||||
model = create_chat_model()
|
||||
tools = [skill_list_tool, skill_view_tool, skill_manage_tool, _make_skill_archive_tool(user_id)]
|
||||
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
|
||||
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
|
||||
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
|
||||
# (wrong action / missing required args), the exception is converted
|
||||
# into a ToolMessage(status="error") so the LLM can read the error and
|
||||
@@ -618,25 +682,26 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> str:
|
||||
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
|
||||
|
||||
messages = result.get("messages", [])
|
||||
tool_actions = _extract_tool_actions(messages)
|
||||
for msg in reversed(messages):
|
||||
if not getattr(msg, "tool_calls", None):
|
||||
content = getattr(msg, "content", "")
|
||||
if isinstance(content, str) and content.strip():
|
||||
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
|
||||
return content
|
||||
return content, tool_actions
|
||||
if isinstance(content, list):
|
||||
text = " ".join(
|
||||
block.get("text", "") if isinstance(block, dict) else str(block)
|
||||
for block in content
|
||||
).strip()
|
||||
curator_log.record("info", "LLM 评审:Agent 已返回最终结论", user_id=user_id)
|
||||
return text
|
||||
return text, tool_actions
|
||||
curator_log.record("warning", "LLM 评审:本次未产出最终回复", user_id=user_id)
|
||||
return ""
|
||||
return "", tool_actions
|
||||
except Exception as e:
|
||||
logger.warning("Curator LLM review failed: %s", e)
|
||||
curator_log.record("warning", f"LLM 评审失败:{e}", user_id=user_id)
|
||||
return ""
|
||||
return "", []
|
||||
|
||||
|
||||
def _parse_structured_summary(text: str) -> dict:
|
||||
@@ -653,6 +718,44 @@ def _parse_structured_summary(text: str) -> dict:
|
||||
return {"consolidations": [], "prunings": []}
|
||||
|
||||
|
||||
def _extract_tool_actions(messages: list) -> list[dict]:
|
||||
"""从 agent 消息流里提取**真实发生的**工具调用 + 结果。
|
||||
|
||||
与 _parse_structured_summary(解析 LLM 自述的 YAML)互补:这里记录的是 curator
|
||||
实际调了哪些工具、入参是什么、成功还是失败 —— 用于审计"嘴上说的"与"实际做的"
|
||||
是否一致。每条形如 ``{tool, args, status, result}``。
|
||||
"""
|
||||
# 先按 tool_call_id 收集每次调用的工具名与入参(来自 AIMessage.tool_calls)
|
||||
calls: dict[str, dict] = {}
|
||||
for msg in messages:
|
||||
for tc in getattr(msg, "tool_calls", None) or []:
|
||||
tc_id = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not tc_id:
|
||||
continue
|
||||
calls[tc_id] = {
|
||||
"tool": tc.get("name") if isinstance(tc, dict) else getattr(tc, "name", None),
|
||||
"args": (tc.get("args") if isinstance(tc, dict) else getattr(tc, "args", None)) or {},
|
||||
}
|
||||
# 再按 ToolMessage(带 tool_call_id)收集真实结果
|
||||
actions: list[dict] = []
|
||||
for msg in messages:
|
||||
tc_id = getattr(msg, "tool_call_id", None)
|
||||
if not tc_id:
|
||||
continue
|
||||
call = calls.get(tc_id, {})
|
||||
raw = getattr(msg, "content", "")
|
||||
result = raw if isinstance(raw, str) else str(raw)
|
||||
actions.append(
|
||||
{
|
||||
"tool": call.get("tool") or getattr(msg, "name", None),
|
||||
"args": call.get("args", {}),
|
||||
"status": getattr(msg, "status", None) or "success",
|
||||
"result": result[:500],
|
||||
}
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
def run_curator_review(user_id: str) -> dict:
|
||||
"""Main curator run for ``user_id``: transitions → LLM consolidation → state.
|
||||
|
||||
@@ -684,16 +787,23 @@ def run_curator_review(user_id: str) -> dict:
|
||||
"info", f"进入 LLM 评审的候选技能:共 {candidate_line_count} 个", user_id=user_id
|
||||
)
|
||||
|
||||
llm_text = _run_llm_review(candidate_list, user_id=user_id)
|
||||
suggestions = _parse_structured_summary(llm_text)
|
||||
curator_log.record(
|
||||
"info",
|
||||
(
|
||||
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
|
||||
f"建议删减 {len(suggestions['prunings'])} 项"
|
||||
),
|
||||
user_id=user_id,
|
||||
)
|
||||
if candidate_line_count == 0:
|
||||
# 没有候选技能 —— 跳过 LLM 评审,省掉无意义的模型调用。
|
||||
# 年龄流转(上面的 apply_automatic_transitions)已完成,且不依赖 LLM。
|
||||
curator_log.record("info", "无候选技能,跳过 LLM 评审", user_id=user_id)
|
||||
suggestions = {"consolidations": [], "prunings": []}
|
||||
tool_actions: list[dict] = []
|
||||
else:
|
||||
llm_text, tool_actions = _run_llm_review(candidate_list, user_id=user_id)
|
||||
suggestions = _parse_structured_summary(llm_text)
|
||||
curator_log.record(
|
||||
"info",
|
||||
(
|
||||
f"LLM 建议已解析:建议合并 {len(suggestions['consolidations'])} 项,"
|
||||
f"建议删减 {len(suggestions['prunings'])} 项"
|
||||
),
|
||||
user_id=user_id,
|
||||
)
|
||||
# Surface per-item reasoning so operators can audit *why* each action
|
||||
# was taken, not just the count.
|
||||
for item in suggestions.get("consolidations", []) or []:
|
||||
@@ -718,15 +828,26 @@ def run_curator_review(user_id: str) -> dict:
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
# 上面是 LLM"自述"的建议;下面记录它"实际"调用了哪些工具,供对照审计。
|
||||
curator_log.record("info", f"LLM 实际调用工具 {len(tool_actions)} 次", user_id=user_id)
|
||||
for act in tool_actions:
|
||||
target = act.get("args", {}).get("name", "")
|
||||
curator_log.record(
|
||||
"info",
|
||||
f" 实际调用 [{act.get('tool')}]{f' [{target}]' if target else ''} → {act.get('status')}",
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
state = _read_curator_state(user_id)
|
||||
state["last_run_at"] = now.isoformat()
|
||||
state["run_count"] = state.get("run_count", 0) + 1
|
||||
state["last_run_summary"] = (
|
||||
f"自动流转:标记陈旧 {transition_counters['marked_stale']} 个、"
|
||||
f"归档 {transition_counters['archived']} 个;"
|
||||
f"LLM:建议合并 {len(suggestions['consolidations'])} 项"
|
||||
f"LLM:建议合并 {len(suggestions['consolidations'])} 项、实际工具调用 {len(tool_actions)} 次"
|
||||
)
|
||||
state["last_consolidation_suggestions"] = suggestions
|
||||
state["last_consolidation_suggestions"] = suggestions # LLM 自述
|
||||
state["last_actual_actions"] = tool_actions # 真实工具调用记录
|
||||
_write_curator_state(state, user_id)
|
||||
|
||||
logger.info("Curator[%s]: run complete — %s", user_id, state["last_run_summary"])
|
||||
@@ -768,10 +889,11 @@ def _curator_thread_target(user_id: str) -> None:
|
||||
|
||||
|
||||
async def maybe_run_curator() -> None:
|
||||
"""Async startup entry — kicks off a thread per user whose interval elapsed.
|
||||
"""周期调度入口 —— 给每个 interval 已到期的用户起一个后台策展线程。
|
||||
|
||||
Skips users with no DB-tracked entries (a fresh deployment) and never
|
||||
blocks lifespan startup on failure.
|
||||
由 app 的 curator 调度器周期性调用(也用于手动触发)。无候选技能的用户,其
|
||||
LLM 评审会在 run_curator_review 内被跳过 —— 只做不耗费 LLM 的年龄流转。
|
||||
任何失败都不会阻断调用方。
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
@@ -819,7 +941,7 @@ def get_curator_status(user_id: str) -> dict:
|
||||
s = entry.get("state", STATE_ACTIVE)
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
|
||||
cfg = _get_curator_config()
|
||||
cfg = _get_curator_config(user_id)
|
||||
return {
|
||||
"enabled": cfg is not None and getattr(cfg, "enabled", True),
|
||||
"run_count": state.get("run_count", 0),
|
||||
@@ -832,4 +954,8 @@ def get_curator_status(user_id: str) -> dict:
|
||||
"stale": counts[STATE_STALE],
|
||||
"archived": counts[STATE_ARCHIVED],
|
||||
},
|
||||
# 上次运行的两份记录:LLM 自述的建议 + 实际发生的工具调用,供前端对照展示。
|
||||
"last_consolidation_suggestions": state.get("last_consolidation_suggestions")
|
||||
or {"consolidations": [], "prunings": []},
|
||||
"last_actual_actions": state.get("last_actual_actions") or [],
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -13,6 +14,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_USAGE_FILENAME = ".usage.json"
|
||||
|
||||
# 串行化对 .usage.json 的"读-改-写",避免并发(尤其 curator 多线程)丢更新。
|
||||
# .usage.json 是全局单文件;此锁覆盖进程内并发。leader 选举已使 curator 只在
|
||||
# 单进程跑,故进程内锁即可保证状态流转不丢。
|
||||
_usage_lock = threading.Lock()
|
||||
|
||||
STATE_ACTIVE = "active"
|
||||
STATE_STALE = "stale"
|
||||
STATE_ARCHIVED = "archived"
|
||||
@@ -75,13 +81,20 @@ def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _update_entry(name: str, updates: dict) -> None:
|
||||
def _update_entry(name: str, updates) -> None:
|
||||
"""更新一个 usage 条目。``updates`` 可以是 dict,或 ``(entry) -> dict`` 回调。
|
||||
|
||||
用回调形式可让"读当前值 -> 算增量"也落在锁内(例如计数自增),
|
||||
避免并发下的 lost update。
|
||||
"""
|
||||
try:
|
||||
data = _read_all()
|
||||
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
|
||||
entry.update(updates)
|
||||
data[name] = entry
|
||||
_write_all(data)
|
||||
with _usage_lock:
|
||||
data = _read_all()
|
||||
entry = {**_DEFAULT_ENTRY, **data.get(name, {})}
|
||||
resolved = updates(entry) if callable(updates) else updates
|
||||
entry.update(resolved)
|
||||
data[name] = entry
|
||||
_write_all(data)
|
||||
except Exception:
|
||||
logger.debug("Failed to update usage entry for %r", name, exc_info=True)
|
||||
|
||||
@@ -103,18 +116,16 @@ def agent_created_entries() -> dict:
|
||||
|
||||
|
||||
def bump_view(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"view_count": entry.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
|
||||
# 用回调:读当前计数 + 自增在同一把锁内完成,避免并发丢计数。
|
||||
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
|
||||
|
||||
|
||||
def bump_use(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"use_count": entry.get("use_count", 0) + 1, "last_used_at": _now_iso()})
|
||||
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
|
||||
|
||||
|
||||
def bump_patch(name: str) -> None:
|
||||
entry = get_entry(name)
|
||||
_update_entry(name, {"patch_count": entry.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
|
||||
_update_entry(name, lambda e: {"patch_count": e.get("patch_count", 0) + 1, "last_patched_at": _now_iso()})
|
||||
|
||||
|
||||
def set_state(name: str, state: str) -> None:
|
||||
@@ -132,9 +143,10 @@ def set_pinned(name: str, pinned: bool) -> None:
|
||||
|
||||
def forget(name: str) -> None:
|
||||
try:
|
||||
data = _read_all()
|
||||
data.pop(name, None)
|
||||
_write_all(data)
|
||||
with _usage_lock:
|
||||
data = _read_all()
|
||||
data.pop(name, None)
|
||||
_write_all(data)
|
||||
except Exception:
|
||||
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
|
||||
|
||||
|
||||
+84
-5
@@ -59,6 +59,66 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_effective_user_id() -> str | None:
|
||||
"""当前请求/线程绑定的用户 id;解析失败时返回 None。"""
|
||||
try:
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
return get_effective_user_id()
|
||||
except Exception:
|
||||
logger.debug("Failed to resolve effective user id for skill_list", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _run_coro_blocking(coro):
|
||||
"""在同步工具里把一个协程跑到完成。
|
||||
|
||||
``skill_list`` 是同步 ``@tool``:LangChain 通常在线程池里执行它(无运行中的
|
||||
事件循环),但 curator 评审会在已有事件循环的线程里调用它。两种情况都要能
|
||||
拿到 ``SkillStore`` 的异步查询结果 —— 检测到运行中的事件循环就转交独立线程
|
||||
执行 ``asyncio.run``,否则直接跑。
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
"""返回 ``owner_user_id == user_id`` 的自定义技能名集合。
|
||||
|
||||
用于默认 lead agent 的可见性过滤:只算"用户自己拥有的"自定义技能,不含
|
||||
其他用户发布的。返回 ``None`` 表示 ``SkillStore`` 不可用(未配置数据库
|
||||
等)—— 调用方应降级为"放行全部自定义技能",避免离线无库部署下技能凭空
|
||||
消失。
|
||||
"""
|
||||
if not user_id:
|
||||
return None
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.skills import make_skill_store
|
||||
|
||||
factory = get_session_factory()
|
||||
if factory is None:
|
||||
return None
|
||||
store = make_skill_store(factory)
|
||||
records = _run_coro_blocking(store.list_visible(user_id))
|
||||
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
|
||||
except Exception:
|
||||
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
@tool("skill_view")
|
||||
def skill_view_tool(name: str) -> str:
|
||||
"""Read the full SKILL.md content of a skill by name.
|
||||
@@ -106,28 +166,47 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
under the ``skills:`` field — the agent only sees what it was wired to
|
||||
use.
|
||||
|
||||
For the **default lead agent** (no agent_id in runtime), there is no
|
||||
per-agent whitelist and every visible skill is returned.
|
||||
For the **default lead agent** (no agent_id in runtime), the result is
|
||||
filtered to non-archived skills that are either built-in (public) or
|
||||
owned by the current user. Other users' published custom skills and
|
||||
archived skills are excluded.
|
||||
|
||||
Returns a JSON array with fields: name, description, category, enabled,
|
||||
state, use_count, view_count, patch_count, last_activity_at, pinned,
|
||||
source.
|
||||
"""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.usage import all_entries, latest_activity_at
|
||||
from deerflow.skills.types import SkillCategory
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
|
||||
|
||||
agent_id = _resolve_active_agent_id(runtime)
|
||||
allowed = _load_agent_skill_whitelist(agent_id)
|
||||
|
||||
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
|
||||
# 当前用户"的技能。owned 为 None 表示 SkillStore 不可用,降级放行全部自定义
|
||||
# 技能(见 _load_owned_skill_names)。
|
||||
owned: set[str] | None = None
|
||||
if allowed is None:
|
||||
owned = _load_owned_skill_names(_resolve_effective_user_id())
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
usage = all_entries()
|
||||
result = []
|
||||
for skill in skills:
|
||||
if allowed is not None and skill.name not in allowed:
|
||||
continue
|
||||
entry = usage.get(skill.name, {})
|
||||
if allowed is not None:
|
||||
# 自定义 agent:严格按 config.yaml 的 skills 白名单过滤。
|
||||
if skill.name not in allowed:
|
||||
continue
|
||||
else:
|
||||
# 默认 agent:跳过已归档技能。
|
||||
if entry.get("state") == STATE_ARCHIVED:
|
||||
continue
|
||||
# 自定义技能仅当属于当前用户时可见;内置 public 技能始终可见。
|
||||
if skill.category == SkillCategory.CUSTOM and owned is not None and skill.name not in owned:
|
||||
continue
|
||||
result.append({
|
||||
"name": skill.name,
|
||||
"description": skill.description,
|
||||
|
||||
@@ -84,12 +84,26 @@ def get_available_tools(
|
||||
builtin_tools = BUILTIN_TOOLS.copy()
|
||||
if not is_scheduled_run:
|
||||
builtin_tools.append(scheduled_task_tool)
|
||||
skill_evolution_config = getattr(config, "skill_evolution", None)
|
||||
# skill_list / skill_view 是只读工具(发现、查看技能),与"自我进化"写能力无关,
|
||||
# 始终提供给 agent。
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
|
||||
builtin_tools.extend([skill_list_tool, skill_view_tool])
|
||||
|
||||
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
|
||||
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
|
||||
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
|
||||
try:
|
||||
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
skill_evolution_config = load_skill_evolution_config_for_user(get_effective_user_id())
|
||||
except Exception: # noqa: BLE001 - 解析失败不能阻断 agent 构建
|
||||
skill_evolution_config = getattr(config, "skill_evolution", None)
|
||||
if getattr(skill_evolution_config, "enabled", False):
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||
from deerflow.tools.skill_manage_tool import skill_manage_tool
|
||||
|
||||
builtin_tools.extend([skill_list_tool, skill_view_tool, skill_manage_tool])
|
||||
builtin_tools.append(skill_manage_tool)
|
||||
|
||||
# Add subagent tools only if enabled via runtime parameter
|
||||
if subagent_enabled:
|
||||
|
||||
Reference in new issue
Block a user