# Conflicts:
#	frontend-web/src/components/workspace/chats/chat-box.tsx
#	frontend-web/src/pages/StrategyRoutes.tsx
#	offline-backend-20260512/backend/app/gateway/routers/agents.py
This commit is contained in:
董宏凯 committed 2026-05-28 15:36:11 +08:00
commit d2f3eba434
81 files changed
+10489 -1653

No files matched your search

@@ -9,6 +9,7 @@ from deerflow.agents.middlewares.clarification_middleware import ClarificationMi
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
@@ -254,11 +255,14 @@ def _build_middlewares(
List of middleware instances.
"""
resolved_app_config = app_config or get_app_config()
cfg = _get_runtime_config(config)
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
# Inject the admin prompt prefix into the model request (kept out of the
# persisted/displayed message so the user never sees it).
middlewares.append(PromptPrefixMiddleware())
if cfg.get("rag_mode_enabled", False):
middlewares.append(RagCitationMiddleware())
# Add summarization middleware if enabled
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
@@ -266,7 +270,6 @@ def _build_middlewares(
middlewares.append(summarization_middleware)
# Add TodoList middleware if plan mode is enabled
cfg = _get_runtime_config(config)
is_plan_mode = cfg.get("is_plan_mode", False)
is_bootstrap = cfg.get("is_bootstrap", False)
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
@@ -371,6 +374,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
writing_mode = bool(cfg.get("writing_mode", False))
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
@@ -461,6 +465,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
is_bootstrap=True,
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
rag_mode_enabled=rag_mode_enabled,
),
state_schema=ThreadState,
)
@@ -495,6 +500,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
rag_mode_enabled=rag_mode_enabled,
is_notebook_mode=is_notebook_mode,
notebook_source_ids=notebook_source_ids,
),
@@ -667,17 +667,25 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
@lru_cache(maxsize=32)
def _get_cached_skills_prompt_section(
skill_signature: tuple[tuple[str, str, str, str], ...],
display_signature: tuple[tuple[str, str, str], ...],
available_skills_key: tuple[str, ...] | None,
container_base_path: str,
skill_evolution_section: str,
) -> str:
display_by_skill = {name: (mode, custom_instruction) for name, mode, custom_instruction in display_signature}
filtered = [(name, description, category, location) for name, description, category, location in skill_signature if available_skills_key is None or name in available_skills_key]
skills_list = ""
if filtered:
skill_items = "\n".join(
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>\n </skill>"
for name, description, category, location in filtered
)
items = []
for name, description, category, location in filtered:
display_mode, custom_instruction = display_by_skill.get(name, ("", ""))
display_hint = ""
if display_mode:
display_hint = f"\n <display mode=\"{display_mode}\">{custom_instruction}</display>"
items.append(
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>{display_hint}\n </skill>"
)
skill_items = "\n".join(items)
skills_list = f"<available_skills>\n{skill_items}\n</available_skills>"
# Restricted mode: agent has an explicit skill allowlist.
@@ -693,6 +701,19 @@ def _get_cached_skills_prompt_section(
else:
scope_note = ""
display_pattern = (
"""
**Skill Result Display Pattern:**
- If a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[[1]]`, the second as `[[2]]`, and so on. Use only numbers that correspond to returned results.
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
"""
if display_signature
else ""
)
return f"""<skill_system>
You have access to skills that provide optimized workflows for specific tasks. Each skill contains best practices, frameworks, and references to additional resources.
{scope_note}
@@ -702,13 +723,19 @@ You have access to skills that provide optimized workflows for specific tasks. E
3. The skill file contains references to external resources under the same folder
4. Load referenced resources only when needed during execution
5. Follow the skill's instructions precisely
{display_pattern}
{location_hint}{evolution_block}
{skills_list}
</skill_system>"""
def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_config: AppConfig | None = None) -> str:
def get_skills_prompt_section(
available_skills: set[str] | None = None,
*,
app_config: AppConfig | None = None,
display_enabled: bool = False,
) -> str:
"""Generate the skills prompt section with available skills list."""
skills = _get_enabled_skills_for_config(app_config)
@@ -739,11 +766,30 @@ def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_c
return ""
skill_signature = tuple((skill.name, skill.description, skill.category, skill.get_container_file_path(container_base_path)) for skill in skills)
display_signature: tuple[tuple[str, str, str], ...] = ()
if display_enabled:
try:
from deerflow.config.extensions_config import get_extensions_config
extensions = get_extensions_config()
display_signature = tuple(
(
name,
skill_config.display.mode,
skill_config.display.custom_instruction,
)
for name, skill_config in sorted(extensions.skills.items())
if skill_config.display is not None
and skill_config.display.enabled
and skill_config.display.mode != "none"
)
except Exception:
logger.debug("Failed to load skill display configs for prompt injection", exc_info=True)
available_key = tuple(sorted(available_skills)) if available_skills is not None else None
if not skill_signature and available_key is not None:
return ""
skill_evolution_section = _build_skill_evolution_section(effective_skill_evolution_enabled)
return _get_cached_skills_prompt_section(skill_signature, available_key, container_base_path, skill_evolution_section)
return _get_cached_skills_prompt_section(skill_signature, display_signature, available_key, container_base_path, skill_evolution_section)
def get_agent_soul(agent_id: str | None) -> str:
@@ -868,6 +914,7 @@ def apply_prompt_template(
is_bootstrap: bool = False,
app_config: AppConfig | None = None,
memory_injection_enabled: bool = True,
rag_mode_enabled: bool = False,
is_notebook_mode: bool = False,
notebook_source_ids: list[int] | None = None,
) -> str:
@@ -902,7 +949,11 @@ def apply_prompt_template(
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
# Get skills section
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
skills_section = get_skills_prompt_section(
available_skills,
app_config=app_config,
display_enabled=rag_mode_enabled,
)
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
@@ -940,6 +991,21 @@ Writing mode is enabled for this run.
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
network_policy_section = _build_network_policy_section()
rag_mode_section = (
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
When a search, knowledge, skill, or MCP tool returns a result list:
- If the skill has a display adapter, follow that adapter first.
- For `citation`, cite returned items with `[[1]]`, `[[2]]`, etc. Only cite real returned items.
- For `table`, do not hand-write the table. Summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the table should appear. The frontend will render the real tool result as a stable table.
- For `list` or `card`, summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="list" />` or `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the structured result should appear.
- If no display adapter exists, default to citation mode with numbered citations.
</rag_citation_mode>
"""
if rag_mode_enabled
else ""
)
# Format the prompt with dynamic skills and memory
prompt = SYSTEM_PROMPT_TEMPLATE.format(
@@ -957,7 +1023,7 @@ Writing mode is enabled for this run.
acp_section=acp_and_mounts_section,
)
prompt = prompt + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
if is_notebook_mode:
source_ids = notebook_source_ids or []
@@ -61,6 +61,21 @@ _AUTH_PATTERNS = (
"无权",
"未授权",
)
# Upstream model gateway returned a malformed payload (typically a JSON decode
# error wrapped as `bad_response_status_code`, or a truncated/garbled inference
# response). These are almost always transient inference-side issues and safe
# to retry.
_UPSTREAM_GARBAGE_PATTERNS = (
"bad_response_status_code",
"bad response status",
"expecting ',' delimiter",
"expecting value",
"expecting property name",
"unexpected character",
"unterminated string",
"extra data",
"jsondecodeerror",
)
class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
@@ -139,9 +154,10 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
detail = _extract_error_detail(exc)
lowered = detail.lower()
error_code = _extract_error_code(exc)
error_code_lower = str(error_code).lower() if error_code is not None else ""
status_code = _extract_status_code(exc)
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(str(error_code).lower(), _QUOTA_PATTERNS):
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(error_code_lower, _QUOTA_PATTERNS):
return False, "quota"
if _matches_any(lowered, _AUTH_PATTERNS):
return False, "auth"
@@ -159,6 +175,11 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
return True, "transient"
if _matches_any(lowered, _BUSY_PATTERNS):
return True, "busy"
# Treat upstream gateway garbage (malformed JSON, truncated responses)
# as a retriable transient — the deterministic char-position errors we
# see from vLLM/proxy layers usually clear on a re-roll.
if _matches_any(lowered, _UPSTREAM_GARBAGE_PATTERNS) or _matches_any(error_code_lower, _UPSTREAM_GARBAGE_PATTERNS):
return True, "upstream"
return False, "generic"
@@ -171,21 +192,34 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
seconds = max(1, round(wait_ms / 1000))
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
if reason == "busy":
reason_text = "服务繁忙"
elif reason == "upstream":
reason_text = "上游响应异常"
else:
reason_text = "网络/连接异常"
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
def _build_circuit_breaker_message(self) -> str:
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
def _build_user_message(self, exc: BaseException, reason: str) -> str:
detail = _extract_error_detail(exc)
def _build_user_message(self, reason: str) -> str:
if reason == "quota":
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
if reason == "auth":
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
if reason in {"busy", "transient"}:
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
return f"大模型因「{detail}」原因调用失败。"
if reason == "upstream":
return "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。"
# NOTE: We intentionally do NOT embed the raw exception detail into the
# user-facing message. Raw `Error code: 400 - {...}` strings — especially
# Python repr-style dict literals — get persisted as AI messages and then
# re-fed to the model on the next turn, where their unescaped quotes /
# braces can corrupt the upstream JSON payload and cause deterministic
# JSONDecodeError loops. Keep the user message clean; full detail is
# already in the backend log via `logger.warning(..., exc_info=exc)`.
return "大模型本次调用未能成功完成,请稍后再试或更换模型。"
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
try:
@@ -249,7 +283,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(exc, reason))
return AIMessage(content=self._build_user_message(reason))
@override
async def awrap_model_call(
@@ -295,7 +329,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(exc, reason))
return AIMessage(content=self._build_user_message(reason))
def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool:
@@ -0,0 +1,109 @@
"""Inject stable citation numbering before RAG final answers."""
from __future__ import annotations
import logging
import os
from pathlib import Path
from collections.abc import Awaitable, Callable
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import SystemMessage
from deerflow.runtime.references import build_latest_reference_prompt, build_reference_debug_summary
logger = logging.getLogger(__name__)
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
def _get_rag_debug_logger() -> logging.Logger:
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
if getattr(debug_logger, "_rag_file_configured", False):
return debug_logger
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
if not log_path.is_absolute():
log_path = Path.cwd() / log_path
log_path.parent.mkdir(parents=True, exist_ok=True)
handler = logging.FileHandler(log_path, encoding="utf-8")
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
debug_logger.addHandler(handler)
debug_logger.setLevel(logging.INFO)
debug_logger.propagate = False
setattr(debug_logger, "_rag_file_configured", True)
setattr(debug_logger, "_rag_file_path", str(log_path))
return debug_logger
def _log_rag_debug(message: str, *args: Any) -> None:
logger.info(message, *args)
_get_rag_debug_logger().info(message, *args)
class RagCitationMiddleware(AgentMiddleware[AgentState]):
"""Give the model the same numbered references the frontend will render."""
@staticmethod
def _enabled(request: ModelRequest) -> bool:
runtime = getattr(request, "runtime", None)
context = getattr(runtime, "context", None) if runtime is not None else None
if isinstance(context, dict) and context.get("rag_mode_enabled"):
return True
config = getattr(runtime, "config", None) if runtime is not None else None
if isinstance(config, dict):
configurable = config.get("configurable")
if isinstance(configurable, dict) and configurable.get("rag_mode_enabled"):
return True
runtime_context = config.get("context")
if isinstance(runtime_context, dict) and runtime_context.get("rag_mode_enabled"):
return True
return False
@staticmethod
def _patch(request: ModelRequest) -> ModelRequest | None:
if not RagCitationMiddleware._enabled(request):
return None
messages = list(request.messages)
prompt, batch = build_latest_reference_prompt(messages)
if not prompt:
_log_rag_debug(
"middleware reference-prompt enabled=true prompt=false messages=%s",
len(messages),
)
return None
summary = build_reference_debug_summary(messages, [batch] if batch else None)
_log_rag_debug(
"middleware reference-prompt enabled=true prompt=true messages=%s tool_messages=%s "
"raw_items=%s normalized_items=%s kept_items=%s latest_batch_sources=%s",
summary["messages"],
summary["tool_messages"],
summary["raw_items"],
summary["normalized_items"],
summary["kept_items"],
len((batch or {}).get("sources") or []),
)
patched_messages = [
*messages,
SystemMessage(content=prompt),
]
return request.override(messages=patched_messages)
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
patched = self._patch(request)
return handler(patched or request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
patched = self._patch(request)
return await handler(patched or request)
@@ -840,10 +840,13 @@ class DeerFlowClient:
current_config = get_extensions_config()
config_data = {
"mcpServers": mcp_servers,
"skills": {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()},
}
config_data = {
"mcpServers": mcp_servers,
"skills": {
name: skill.model_dump(by_alias=True, exclude_none=True)
for name, skill in current_config.skills.items()
},
}
self._atomic_write_json(config_path, config_data)
@@ -903,13 +906,20 @@ class DeerFlowClient:
if config_path is None:
raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.")
extensions_config = get_extensions_config()
extensions_config.skills[name] = SkillStateConfig(enabled=enabled)
config_data = {
"mcpServers": {n: s.model_dump() for n, s in extensions_config.mcp_servers.items()},
"skills": {n: {"enabled": sc.enabled} for n, sc in extensions_config.skills.items()},
}
extensions_config = get_extensions_config()
current_skill_config = extensions_config.skills.get(name)
extensions_config.skills[name] = SkillStateConfig(
enabled=enabled,
display=current_skill_config.display if current_skill_config else None,
)
config_data = {
"mcpServers": {n: s.model_dump(by_alias=True) for n, s in extensions_config.mcp_servers.items()},
"skills": {
n: sc.model_dump(by_alias=True, exclude_none=True)
for n, sc in extensions_config.skills.items()
},
}
self._atomic_write_json(config_path, config_data)
@@ -3,7 +3,7 @@
import json
import os
from pathlib import Path
from typing import Any, Literal
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -53,10 +53,63 @@ class McpServerConfig(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
class SkillDisplayTableColumn(BaseModel):
"""A configured table column for structured skill result display."""
label: str = Field(description="Column label shown in the frontend table")
field: str = Field(description="Field name or dot path read from each result item")
width: str | None = Field(default=None, description="Optional CSS width for this column")
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayConfig(BaseModel):
"""Frontend display adapter for structured skill results."""
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="none",
description="How the frontend should render this skill's structured results",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="doubleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
class ExtensionsConfig(BaseModel):
@@ -17,6 +17,7 @@ import json
import logging
import threading
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -37,10 +38,53 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class AppearanceSettings(BaseModel):
"""System-wide appearance defaults that any *new* user inherits.
Once a user explicitly toggles their theme in the UI, that choice is
persisted in the browser and supersedes this default — so changing
``default_theme`` here only affects users who have never touched the
toggle, and any future first-time loads.
"""
default_theme: Literal["light", "dark"] = Field(
default="light",
description="Theme applied for users who haven't picked their own (light / dark)",
)
model_config = ConfigDict(extra="ignore")
class CitationDisplaySettings(BaseModel):
"""System-wide cleanup rules for citation cards and detail dialogs."""
cleanup_words: list[str] = Field(
default_factory=list,
description="Words or phrases used to remove noisy advertisement/disclaimer lines from citation display.",
)
model_config = ConfigDict(extra="ignore")
class PrivateSquareSettings(BaseModel):
"""Admin-configurable settings for the agent private-square (私密广场).
The password gates read access to all agents flagged ``is_private=True``.
The hash is stored opaquely — the router layer is responsible for hashing
and verifying so the harness package keeps zero dependencies on
``app.gateway.auth``.
"""
enabled: bool = Field(default=False, description="Whether the private square is offered to users at all")
password_hash: str = Field(default="", description="Opaque hash of the admin-configured password. Empty = no password set yet.")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
model_config = ConfigDict(extra="ignore")
@@ -81,3 +81,39 @@ class AgentStore(ABC):
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
raise NotImplementedError
@abstractmethod
async def list_private(self) -> list[dict[str, Any]]:
"""Return every agent flagged ``is_private=True``.
Caller is responsible for authorization — typically the caller has
already verified the admin-configured private-square password.
"""
raise NotImplementedError
@abstractmethod
async def list_all(self) -> list[dict[str, Any]]:
"""Return every agent regardless of ownership/visibility.
Caller is responsible for authorization — typically the caller has
already verified ``system_role == "admin"``.
"""
raise NotImplementedError
@abstractmethod
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns ``None``
if the agent does not exist.
"""
raise NotImplementedError
@abstractmethod
async def delete_any(self, agent_id: str) -> bool:
"""Delete any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns
``False`` if the agent does not exist.
"""
raise NotImplementedError
@@ -138,3 +138,29 @@ class MemoryAgentStore(AgentStore):
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
@@ -21,6 +21,12 @@ class AgentRow(Base):
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# When True, the agent is moved into the password-gated "private square"
# (私密广场) and is hidden from the regular public square / built-in list.
# The flag is independent of ``published``: a private agent is still
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
@@ -5,7 +5,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, or_, select
from sqlalchemy import and_, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.agents.base import AgentStore
@@ -28,10 +28,31 @@ class AgentRepository(AgentStore):
@staticmethod
def _visible_filter(user_id: str):
"""Visibility for single-record fetches (``get_visible``).
Lenient: a caller who already knows the agent id can fetch it even
when it's flagged ``is_private``. The private-square password is the
*discovery* gate (listings); individual reads are not re-gated, so a
bookmarked / linked private agent stays openable.
"""
return or_(AgentRow.user_id.is_(None), AgentRow.user_id == user_id, AgentRow.published.is_(True))
@staticmethod
def _listing_filter(user_id: str):
"""Visibility for *listings* — hides private-square agents from
non-owners so they only surface inside the password-gated endpoint.
Owners keep seeing their own private agents in '我的'.
"""
return or_(
AgentRow.user_id == user_id,
and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
),
)
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(self._visible_filter(user_id)).order_by(AgentRow.created_at.asc())
stmt = select(AgentRow).where(self._listing_filter(user_id)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
@@ -61,6 +82,7 @@ class AgentRepository(AgentStore):
description=data.get("description") or "",
user_id=data.get("user_id"),
published=bool(data.get("published", False)),
is_private=bool(data.get("is_private", False)),
created_at=now,
updated_at=now,
)
@@ -99,7 +121,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id != user_id:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -112,7 +134,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id is not None:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -157,7 +179,7 @@ class AgentRepository(AgentStore):
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
stmt = (
select(AgentRow)
.where(AgentRow.featured_order.is_not(None), self._visible_filter(user_id))
.where(AgentRow.featured_order.is_not(None), self._listing_filter(user_id))
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
)
async with self._sf() as session:
@@ -168,9 +190,43 @@ class AgentRepository(AgentStore):
if not agent_ids:
return []
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._visible_filter(user_id))
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._listing_filter(user_id))
async with self._sf() as session:
result = await session.execute(stmt)
rows = list(result.scalars())
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
return [_row_to_dict(r) for r in rows]
async def list_private(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(AgentRow.is_private.is_(True)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_all(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete_any(self, agent_id: str) -> bool:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
@@ -0,0 +1,62 @@
"""Add ``agents.is_private`` for the private-square (私密广场) feature.
A nullable-default bool column flagging agents that should live in the
password-gated private square instead of the regular public square.
Revision ID: 20260526_01
Revises: 20260522_01
Create Date: 2026-05-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260526_01"
down_revision = "20260522_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table("agents") and not _has_column("agents", "is_private"):
op.add_column(
"agents",
sa.Column(
"is_private",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
)
if _has_table("agents") and not _has_index("agents", "ix_agents_is_private"):
op.create_index("ix_agents_is_private", "agents", ["is_private"])
def downgrade() -> None:
if _has_index("agents", "ix_agents_is_private"):
op.drop_index("ix_agents_is_private", table_name="agents")
if _has_column("agents", "is_private"):
op.drop_column("agents", "is_private")
@@ -30,7 +30,7 @@ from deerflow.persistence.scheduled_tasks.model import (
ScheduledTaskSubscriptionRow,
SchedulerThreadRow,
)
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
@@ -51,6 +51,7 @@ __all__ = [
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SkillRow",
"SkillDisplayAdapterRow",
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
@@ -68,6 +68,14 @@ class SkillStore(ABC):
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def list_all(self) -> list[dict[str, Any]]:
"""Return every skill record regardless of ownership/visibility.
Caller is responsible for authorization (admin only).
"""
raise NotImplementedError
@abstractmethod
async def list_published_names(self) -> set[str]:
"""Return the set of skill names with ``published = True`` (any owner).
@@ -134,5 +134,11 @@ class MemorySkillStore(SkillStore):
async def count_owned(self, user_id: str) -> int:
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
async def list_all(self) -> list[dict[str, Any]]:
return [
_with_derived(_copy(record)) # type: ignore[arg-type]
for record in self._skills.values()
]
async def list_published_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("published")}
@@ -31,3 +31,16 @@ class SkillRow(Base):
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class SkillDisplayAdapterRow(Base):
__tablename__ = "skill_display_adapters"
skill_name: Mapped[str] = mapped_column(String(64), primary_key=True)
# Store JSON as text for maximum MySQL compatibility. Some deployments still
# run MySQL variants where native JSON columns or unquoted names such as
# ``display`` are not accepted consistently.
display_config: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -149,6 +149,12 @@ class SkillRepository(SkillStore):
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def list_all(self) -> list[dict[str, Any]]:
stmt = select(SkillRow).order_by(SkillRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_published_names(self) -> set[str]:
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
async with self._sf() as session:
@@ -0,0 +1,747 @@
"""Build stable per-turn citation batches from LangGraph messages."""
from __future__ import annotations
import json
import re
from typing import Any
from urllib.parse import quote
from deerflow.config.extensions_config import SkillDisplayConfig, get_extensions_config
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
batches: list[dict[str, Any]] = []
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
sources = _collect_sources(messages[index + 1 : end])
if sources:
anchor = _find_reference_batch_anchor(messages, index + 1, end)
message_id = _message_id(anchor) or _message_id(message) or str(index)
batches.append(
{
"id": message_id,
"assistant_message_id": message_id,
"sources": sources,
}
)
return batches
def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, Any] | None]:
human_index = _find_latest_human_index(messages)
if human_index is None:
return "", None
sources = _collect_sources(messages[human_index + 1 :])
if not sources:
return "", None
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
message_id = _message_id(assistant) or _message_id(messages[human_index]) or str(human_index)
batch = {
"id": message_id,
"assistant_message_id": message_id,
"sources": sources,
}
lines = [
"本轮检索结果已由系统统一合并并重新编号。最终回答必须使用下面这套编号。",
"如果你仍需要补充检索,请先继续调用工具;最终回答只能引用最新合并后的编号。",
"",
]
for source in sources:
title = source.get("title") or f"参考文献 {source.get('index')}"
meta = " · ".join(
str(item)
for item in (source.get("source"), source.get("time"))
if item
)
content = str(source.get("content") or source.get("snippet") or "").strip()
if len(content) > 520:
content = content[:517] + "..."
head = f"[{source.get('index')}] {title}"
if meta:
head += f"({meta})"
lines.append(head)
if content:
lines.append(content)
lines.append("")
lines.extend(
[
"引用规则:",
"1. 回答正文中的关键结论必须标注对应编号,例如 [1]、[2]。",
"2. 多个来源可写为 [1][3][5]。",
"3. 只能引用上方存在的编号,不要编造编号。",
"4. 不要使用工具原始编号,也不要重新排序编号。",
"5. 不要在正文末尾堆完整参考文献列表,前端会展示右侧参考文献。",
]
)
return "\n".join(lines).strip(), batch
def _find_latest_human_index(messages: list[Any]) -> int | None:
for index in range(len(messages) - 1, -1, -1):
if _message_type(messages[index]) == "human":
return index
return None
def build_reference_debug_summary(messages: list[Any], batches: list[dict[str, Any]] | None = None) -> dict[str, Any]:
"""Return compact counts for citation extraction logs."""
turn_summaries: list[dict[str, Any]] = []
total_tool_messages = 0
total_raw_items = 0
total_normalized_items = 0
total_kept_items = 0
for index, message in enumerate(messages):
if _message_type(message) != "human":
continue
end = len(messages)
for cursor in range(index + 1, len(messages)):
if _message_type(messages[cursor]) == "human":
end = cursor
break
tool_messages = 0
raw_items = 0
normalized_items = 0
kept_items = 0
for tool_message in messages[index + 1 : end]:
if _message_type(tool_message) != "tool":
continue
tool_messages += 1
items = _first_list(_extract_tool_payload(tool_message))
raw_items += len(items)
for raw_item in items:
normalized = _normalize_reference_item(raw_item, normalized_items + 1)
if not normalized:
continue
normalized_items += 1
kept_items += 1
total_tool_messages += tool_messages
total_raw_items += raw_items
total_normalized_items += normalized_items
total_kept_items += kept_items
turn_summaries.append(
{
"human_index": index,
"tool_messages": tool_messages,
"raw_items": raw_items,
"normalized_items": normalized_items,
"kept_items": kept_items,
}
)
batch_summaries = [
{
"id": batch.get("id"),
"assistant_message_id": batch.get("assistant_message_id"),
"source_count": len(batch.get("sources") or []),
}
for batch in (batches or build_reference_batches(messages))
]
return {
"messages": len(messages),
"turns": len(turn_summaries),
"tool_messages": total_tool_messages,
"raw_items": total_raw_items,
"normalized_items": total_normalized_items,
"kept_items": total_kept_items,
"turns_detail": turn_summaries,
"batches": batch_summaries,
}
def adapt_reference_batches_with_displays(
batches: list[dict[str, Any]],
displays: dict[str, SkillDisplayConfig | dict[str, Any]],
) -> list[dict[str, Any]]:
"""Apply persisted skill display configs to already-built reference batches.
The reference builder can run before the gateway has a fresh extensions
cache, while the skill adapter editor persists the user's latest mapping in
the database. This pass lets the API response use that DB-backed mapping
without changing citation indexes.
"""
normalized_displays: dict[str, SkillDisplayConfig] = {}
for skill_name, display in displays.items():
try:
config = (
display
if isinstance(display, SkillDisplayConfig)
else SkillDisplayConfig.model_validate(display)
)
except Exception:
continue
if not config.enabled or config.mode == "none":
continue
normalized_displays[skill_name] = config
if not normalized_displays:
return batches
displays_by_normalized_name = {
_normalize_name(name): (name, display)
for name, display in normalized_displays.items()
}
adapted_batches: list[dict[str, Any]] = []
for batch in batches:
sources: list[dict[str, Any]] = []
for source in batch.get("sources", []):
if not isinstance(source, dict):
continue
skill_name = str(source.get("skillName") or "")
display = normalized_displays.get(skill_name)
if display is None:
normalized_source = _normalize_name(skill_name)
normalized_match = displays_by_normalized_name.get(normalized_source)
if normalized_match is None:
for normalized_skill, candidate in displays_by_normalized_name.items():
if (
normalized_source.endswith(normalized_skill)
or normalized_skill in normalized_source
):
normalized_match = candidate
break
if normalized_match is not None:
skill_name, display = normalized_match
raw = source.get("raw")
if display is not None and isinstance(raw, dict):
index_value = int(source.get("index") or len(sources) + 1)
configured = _normalize_with_display_config(
raw,
index_value,
skill_name,
display,
)
if configured is not None:
sources.append(
{
**source,
**configured,
"index": index_value,
"skillName": skill_name,
"raw": raw,
}
)
continue
sources.append(source)
adapted_batches.append({**batch, "sources": sources})
return adapted_batches
def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
sources: list[dict[str, Any]] = []
pending_tool_contexts: list[dict[str, Any]] = []
tool_contexts_by_id: dict[str, dict[str, Any]] = {}
for message in messages:
message_type = _message_type(message)
if message_type == "ai":
for tool_call in _extract_tool_calls(message):
context = _build_tool_context(tool_call)
pending_tool_contexts.append(context)
if context.get("id"):
tool_contexts_by_id[str(context["id"])] = context
continue
if message_type != "tool":
continue
tool_context = _resolve_tool_context(
message,
pending_tool_contexts,
tool_contexts_by_id,
)
payload = _extract_tool_payload(message)
for raw_item in _first_list(payload):
normalized = _normalize_reference_item(
raw_item,
len(sources) + 1,
payload=payload,
tool_context=tool_context,
)
if not normalized:
continue
normalized["index"] = len(sources) + 1
sources.append(normalized)
return sources
def _find_reference_batch_anchor(messages: list[Any], start: int, end: int) -> Any | None:
answer = _find_last_assistant_answer(messages, start, end)
if answer is not None:
return answer
for index in range(end - 1, start - 1, -1):
candidate = messages[index]
if _message_type(candidate) == "ai" and _message_id(candidate):
return candidate
return None
def _find_last_assistant_answer(messages: list[Any], start: int, end: int) -> Any | None:
for index in range(end - 1, start - 1, -1):
candidate = messages[index]
if (
_message_type(candidate) == "ai"
and _message_id(candidate)
and _has_answer_content(candidate)
):
return candidate
return None
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "")
return str(getattr(message, "type", "") or "")
def _message_id(message: Any) -> str | None:
value = message.get("id") if isinstance(message, dict) else getattr(message, "id", None)
return str(value) if value else None
def _message_content(message: Any) -> Any:
return message.get("content") if isinstance(message, dict) else getattr(message, "content", None)
def _message_name(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
return str(getattr(message, "name", None) or "tool")
def _message_tool_call_id(message: Any) -> str | None:
if isinstance(message, dict):
value = message.get("tool_call_id") or message.get("toolCallId")
else:
value = getattr(message, "tool_call_id", None) or getattr(message, "toolCallId", None)
return str(value) if value else None
def _has_tool_calls(message: Any) -> bool:
if isinstance(message, dict):
direct = message.get("tool_calls")
chunks = message.get("tool_call_chunks")
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
else:
direct = getattr(message, "tool_calls", None)
chunks = getattr(message, "tool_call_chunks", None)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
return bool(direct or chunks or extra)
def _has_answer_content(message: Any) -> bool:
content = _message_content(message)
if isinstance(content, str):
return bool(content.strip())
if isinstance(content, list):
return any(
isinstance(part, dict)
and part.get("type") == "text"
and str(part.get("text") or "").strip()
for part in content
)
return False
def _extract_tool_calls(message: Any) -> list[dict[str, Any]]:
candidates: list[Any] = []
if isinstance(message, dict):
for key in ("tool_calls", "tool_call_chunks"):
value = message.get(key)
if isinstance(value, list):
candidates.extend(value)
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
if isinstance(extra, list):
candidates.extend(extra)
else:
for key in ("tool_calls", "tool_call_chunks"):
value = getattr(message, key, None)
if isinstance(value, list):
candidates.extend(value)
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
if isinstance(extra, list):
candidates.extend(extra)
calls: list[dict[str, Any]] = []
seen_ids: set[str] = set()
for raw in candidates:
call = _normalize_tool_call(raw)
if not call:
continue
call_id = str(call.get("id") or "")
if call_id and call_id in seen_ids:
continue
if call_id:
seen_ids.add(call_id)
calls.append(call)
return calls
def _normalize_tool_call(raw: Any) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
fn = raw.get("function") if isinstance(raw.get("function"), dict) else {}
name = raw.get("name") or fn.get("name") or raw.get("tool_name")
if not name:
return None
return {
"id": raw.get("id") or raw.get("tool_call_id") or raw.get("call_id"),
"name": str(name),
"args": _normalize_tool_args(raw.get("args") or raw.get("arguments") or fn.get("arguments")),
}
def _normalize_tool_args(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
if isinstance(parsed, dict):
return parsed
except Exception:
pass
return {"command": value}
return {}
def _build_tool_context(tool_call: dict[str, Any]) -> dict[str, Any]:
tool_name = str(tool_call.get("name") or "tool")
args = tool_call.get("args") if isinstance(tool_call.get("args"), dict) else {}
return {
"id": tool_call.get("id"),
"tool_name": tool_name,
"args": args,
"skill_name": _infer_skill_name(tool_name, args),
}
def _resolve_tool_context(
message: Any,
pending_tool_contexts: list[dict[str, Any]],
tool_contexts_by_id: dict[str, dict[str, Any]],
) -> dict[str, Any]:
tool_call_id = _message_tool_call_id(message)
if tool_call_id and tool_call_id in tool_contexts_by_id:
context = tool_contexts_by_id[tool_call_id]
pending_tool_contexts[:] = [
item for item in pending_tool_contexts if str(item.get("id") or "") != tool_call_id
]
return context
if pending_tool_contexts:
return pending_tool_contexts.pop(0)
tool_name = _message_name(message)
return {
"id": tool_call_id,
"tool_name": tool_name,
"args": {},
"skill_name": _infer_skill_name(tool_name, {}),
}
def _extract_tool_payload(message: Any) -> Any:
content = _message_content(message)
if isinstance(content, dict):
return _unwrap_tool_payload(content)
if isinstance(content, list):
texts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
if isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "json" and isinstance(part.get("json"), dict):
return _unwrap_tool_payload(part["json"])
if part.get("type") == "text" and isinstance(part.get("text"), str):
texts.append(part["text"])
unwrapped = _unwrap_tool_payload(part)
if unwrapped is not None:
return unwrapped
content = "\n".join(texts)
if isinstance(content, str):
return _parse_json_like(content)
return None
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
list_keys = ("results", "data", "items", "hits", "documents", "records", "sources")
if any(isinstance(value.get(key), list) for key in list_keys):
return value
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
nested = value.get(key)
if isinstance(nested, str):
parsed = _parse_json_like(nested)
if parsed is not None:
return parsed
if isinstance(nested, dict):
unwrapped = _unwrap_tool_payload(nested)
if unwrapped is not None:
return unwrapped
return value
def _parse_json_like(text: str) -> Any:
trimmed = text.strip()
if not trimmed:
return None
try:
return json.loads(trimmed)
except Exception:
pass
first = trimmed.find("{")
last = trimmed.rfind("}")
if 0 <= first < last:
try:
return json.loads(trimmed[first : last + 1])
except Exception:
return None
return None
def _first_list(payload: Any) -> list[Any]:
if isinstance(payload, list):
return payload
if not isinstance(payload, dict):
return []
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
value = payload.get(key)
if isinstance(value, list):
return value
for key in ("result", "payload", "response", "output", "data"):
found = _first_list(payload.get(key))
if found:
return found
return []
def _nested_records(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, dict):
return []
layers = [raw]
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
nested = raw.get(key)
if isinstance(nested, dict):
layers.append(nested)
return layers
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
for layer in layers:
for key in keys:
value = layer.get(key)
if isinstance(value, (str, int, float)) and str(value).strip():
return str(value).strip()
return None
def _enabled_skill_displays() -> dict[str, SkillDisplayConfig]:
try:
return {
name: state.display
for name, state in get_extensions_config().skills.items()
if state.display is not None
and state.enabled
and state.display.enabled
and state.display.mode != "none"
}
except Exception:
return {}
def _infer_skill_name(tool_name: str, args: dict[str, Any]) -> str | None:
displays = _enabled_skill_displays()
if tool_name in displays:
return tool_name
normalized_tool = _normalize_name(tool_name)
for skill_name in displays:
normalized_skill = _normalize_name(skill_name)
if (
normalized_tool == normalized_skill
or normalized_tool.endswith(normalized_skill)
or normalized_skill in normalized_tool
):
return skill_name
for value in args.values():
if not isinstance(value, str):
continue
match = _SKILL_PATH_RE.search(value)
if match:
return match.group(1)
return None
def _resolve_display_match(
raw: dict[str, Any],
payload: Any,
tool_context: dict[str, Any] | None,
) -> tuple[str, SkillDisplayConfig] | None:
displays = _enabled_skill_displays()
if not displays:
return None
skill_name = str((tool_context or {}).get("skill_name") or "")
display = displays.get(skill_name)
if display is not None:
return skill_name, display
tool_name = str((tool_context or {}).get("tool_name") or "")
inferred = _infer_skill_name(tool_name, (tool_context or {}).get("args") or {})
if inferred and inferred in displays:
return inferred, displays[inferred]
return None
def _normalize_with_display_config(
raw: dict[str, Any],
fallback_index: int,
skill_name: str,
display: SkillDisplayConfig,
) -> dict[str, Any] | None:
title = _stringify(_get_by_path(raw, display.title_field), f"结果 {fallback_index}")
snippet = _stringify(_get_by_path(raw, display.summary_field), "")
content = _stringify(_get_by_path(raw, display.content_field), snippet)
source = _stringify(_get_by_path(raw, display.source_field), "") or None
item_type = _stringify(_get_by_path(raw, display.type_field), "result") or "result"
time = _stringify(_get_by_path(raw, display.time_field), "") or None
item_id = _stringify(_get_by_path(raw, display.id_field), "") or None
score = _number_or_none(_get_by_path(raw, display.score_field))
url = _resolve_display_link(raw, display)
if not title and not content and not url:
return None
return {
"id": item_id or url,
"skillName": skill_name,
"mode": display.mode,
"title": title,
"type": item_type,
"snippet": snippet,
"content": content,
"source": source,
"time": time,
"url": url,
"score": score,
"tableColumns": [column.model_dump(by_alias=True) for column in display.table_columns],
"raw": raw,
}
def _resolve_display_link(raw: dict[str, Any], display: SkillDisplayConfig) -> str | None:
url = _stringify(_get_by_path(raw, display.url_field), "")
item_id = _stringify(_get_by_path(raw, display.id_field), "")
if display.link_strategy == "url":
return url or None
if display.link_strategy == "id":
return _render_template(display.link_template, raw) if item_id and display.link_template else None
return url or (_render_template(display.link_template, raw) if item_id and display.link_template else None) or None
def _render_template(template: str | None, raw: dict[str, Any]) -> str | None:
if not template:
return None
def replace(match: re.Match[str]) -> str:
value = _get_by_path(raw, match.group(1).strip())
return quote(_stringify(value, ""), safe="")
return re.sub(r"\{([^}]+)\}", replace, template)
def _get_by_path(value: Any, path: str | None) -> Any:
if not path:
return None
cursor = value
for part in path.split("."):
if isinstance(cursor, dict):
cursor = cursor.get(part)
else:
return None
return cursor
def _stringify(value: Any, fallback: str = "") -> str:
if value is None:
return fallback
if isinstance(value, (str, int, float)):
text = str(value).strip()
return text if text else fallback
return fallback
def _number_or_none(value: Any) -> float | None:
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
try:
return float(value)
except Exception:
return None
return None
def _normalize_name(name: str) -> str:
return re.sub(r"[^a-z0-9]+", "", name.lower())
def _normalize_reference_item(
raw: Any,
fallback_index: int,
*,
payload: Any = None,
tool_context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
display_match = _resolve_display_match(raw, payload, tool_context)
if display_match is not None:
skill_name, display = display_match
configured = _normalize_with_display_config(raw, fallback_index, skill_name, display)
if configured is not None:
return configured
layers = _nested_records(raw)
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
score_raw = _pick(layers, ("score", "relevance", "similarity"))
score = None
if score_raw is not None:
try:
score = float(score_raw)
except Exception:
score = None
if not title:
title = source or f"结果 {fallback_index}"
if not content:
content = ""
if not title and not content and not url:
return None
return {
"id": item_id or url,
"skillName": str((tool_context or {}).get("skill_name") or (tool_context or {}).get("tool_name") or "backend-reference"),
"mode": "citation",
"title": title,
"type": "result",
"snippet": content,
"content": content,
"source": source,
"time": time,
"url": url,
"score": score,
"raw": raw,
}
def _reference_key(source: dict[str, Any]) -> str:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
item_id = source.get("id") or source.get("url") or raw.get("recUuid") or raw.get("uuid") or raw.get("docId") or raw.get("id")
if item_id:
return str(item_id).casefold().strip()
return "|".join(
str(source.get(key) or "").casefold().strip()
for key in ("title", "source", "time", "content", "snippet")
)
@@ -37,14 +37,26 @@ class LocalSandbox(Sandbox):
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
@@ -302,30 +314,39 @@ class LocalSandbox(Sandbox):
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [shell, "-NoProfile", "-Command", resolved_command]
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
timeout=600,
)
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
output = result.stdout
if result.stderr:
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
@@ -149,3 +149,58 @@ async def cascade_skill_unpublished_or_deleted(
counters["notified"] += 1
return counters
def sync_agent_skills_to_available(available_skill_names: set[str]) -> dict[str, Any]:
"""Strip skill references that no longer exist on disk from every agent.
Walks every agent dir and rewrites its ``config.yaml`` ``skills`` list to
contain only names that appear in ``available_skill_names``. Used by the
admin "agent-skills sync" endpoint to repair historical drift — skills
deleted before the cascade existed, deleted via a path that skipped the
cascade, or removed from the filesystem manually.
Synchronous because it only touches local files and the admin endpoint
runs it in a worker thread.
Returns ``{checked, modified, removals: [{agent_id, removed_skills}]}``.
"""
counters: dict[str, Any] = {"checked": 0, "modified": 0, "removals": []}
for agent_dir in _list_agent_dirs():
config_file = agent_dir / "config.yaml"
if not config_file.exists():
continue
counters["checked"] += 1
try:
with config_file.open("r", encoding="utf-8") as f:
data: dict[str, Any] = yaml.safe_load(f) or {}
except yaml.YAMLError:
logger.warning("Sync: failed to parse %s, skipping", config_file)
continue
skills = data.get("skills")
if not isinstance(skills, list) or not skills:
continue
kept: list[str] = []
removed: list[str] = []
for name in skills:
if isinstance(name, str) and name in available_skill_names:
kept.append(name)
else:
removed.append(name if isinstance(name, str) else str(name))
if not removed:
continue
data["skills"] = kept
with config_file.open("w", encoding="utf-8") as f:
yaml.safe_dump(data, f, default_flow_style=False, allow_unicode=True)
counters["modified"] += 1
counters["removals"].append({"agent_id": agent_dir.name, "removed_skills": removed})
logger.info("Sync: removed orphan skills %s from agent '%s'", removed, agent_dir.name)
return counters