Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into dhk-1
# Conflicts: # frontend-web/src/components/workspace/chats/chat-box.tsx # frontend-web/src/pages/StrategyRoutes.tsx # offline-backend-20260512/backend/app/gateway/routers/agents.py
This commit is contained in:
commit
d2f3eba434
81 files changed
+10489
-1653
No files matched your search
+7
-1
@@ -9,6 +9,7 @@ from deerflow.agents.middlewares.clarification_middleware import ClarificationMi
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
|
||||
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
|
||||
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
|
||||
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
|
||||
@@ -254,11 +255,14 @@ def _build_middlewares(
|
||||
List of middleware instances.
|
||||
"""
|
||||
resolved_app_config = app_config or get_app_config()
|
||||
cfg = _get_runtime_config(config)
|
||||
middlewares = build_lead_runtime_middlewares(app_config=resolved_app_config, lazy_init=True)
|
||||
|
||||
# Inject the admin prompt prefix into the model request (kept out of the
|
||||
# persisted/displayed message so the user never sees it).
|
||||
middlewares.append(PromptPrefixMiddleware())
|
||||
if cfg.get("rag_mode_enabled", False):
|
||||
middlewares.append(RagCitationMiddleware())
|
||||
|
||||
# Add summarization middleware if enabled
|
||||
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
|
||||
@@ -266,7 +270,6 @@ def _build_middlewares(
|
||||
middlewares.append(summarization_middleware)
|
||||
|
||||
# Add TodoList middleware if plan mode is enabled
|
||||
cfg = _get_runtime_config(config)
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
|
||||
@@ -371,6 +374,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
|
||||
rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
|
||||
writing_mode = bool(cfg.get("writing_mode", False))
|
||||
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
|
||||
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
|
||||
@@ -461,6 +465,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
is_bootstrap=True,
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
@@ -495,6 +500,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
),
|
||||
|
||||
+74
-8
@@ -667,17 +667,25 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
@lru_cache(maxsize=32)
|
||||
def _get_cached_skills_prompt_section(
|
||||
skill_signature: tuple[tuple[str, str, str, str], ...],
|
||||
display_signature: tuple[tuple[str, str, str], ...],
|
||||
available_skills_key: tuple[str, ...] | None,
|
||||
container_base_path: str,
|
||||
skill_evolution_section: str,
|
||||
) -> str:
|
||||
display_by_skill = {name: (mode, custom_instruction) for name, mode, custom_instruction in display_signature}
|
||||
filtered = [(name, description, category, location) for name, description, category, location in skill_signature if available_skills_key is None or name in available_skills_key]
|
||||
skills_list = ""
|
||||
if filtered:
|
||||
skill_items = "\n".join(
|
||||
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>\n </skill>"
|
||||
for name, description, category, location in filtered
|
||||
)
|
||||
items = []
|
||||
for name, description, category, location in filtered:
|
||||
display_mode, custom_instruction = display_by_skill.get(name, ("", ""))
|
||||
display_hint = ""
|
||||
if display_mode:
|
||||
display_hint = f"\n <display mode=\"{display_mode}\">{custom_instruction}</display>"
|
||||
items.append(
|
||||
f" <skill>\n <name>{name}</name>\n <description>{description} {_skill_mutability_label(category)}</description>\n <location>{location}</location>{display_hint}\n </skill>"
|
||||
)
|
||||
skill_items = "\n".join(items)
|
||||
skills_list = f"<available_skills>\n{skill_items}\n</available_skills>"
|
||||
|
||||
# Restricted mode: agent has an explicit skill allowlist.
|
||||
@@ -693,6 +701,19 @@ def _get_cached_skills_prompt_section(
|
||||
else:
|
||||
scope_note = ""
|
||||
|
||||
display_pattern = (
|
||||
"""
|
||||
**Skill Result Display Pattern:**
|
||||
- If a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[[1]]`, the second as `[[2]]`, and so on. Use only numbers that correspond to returned results.
|
||||
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
|
||||
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
|
||||
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
|
||||
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
|
||||
"""
|
||||
if display_signature
|
||||
else ""
|
||||
)
|
||||
|
||||
return f"""<skill_system>
|
||||
You have access to skills that provide optimized workflows for specific tasks. Each skill contains best practices, frameworks, and references to additional resources.
|
||||
{scope_note}
|
||||
@@ -702,13 +723,19 @@ You have access to skills that provide optimized workflows for specific tasks. E
|
||||
3. The skill file contains references to external resources under the same folder
|
||||
4. Load referenced resources only when needed during execution
|
||||
5. Follow the skill's instructions precisely
|
||||
{display_pattern}
|
||||
{location_hint}{evolution_block}
|
||||
{skills_list}
|
||||
|
||||
</skill_system>"""
|
||||
|
||||
|
||||
def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_config: AppConfig | None = None) -> str:
|
||||
def get_skills_prompt_section(
|
||||
available_skills: set[str] | None = None,
|
||||
*,
|
||||
app_config: AppConfig | None = None,
|
||||
display_enabled: bool = False,
|
||||
) -> str:
|
||||
"""Generate the skills prompt section with available skills list."""
|
||||
skills = _get_enabled_skills_for_config(app_config)
|
||||
|
||||
@@ -739,11 +766,30 @@ def get_skills_prompt_section(available_skills: set[str] | None = None, *, app_c
|
||||
return ""
|
||||
|
||||
skill_signature = tuple((skill.name, skill.description, skill.category, skill.get_container_file_path(container_base_path)) for skill in skills)
|
||||
display_signature: tuple[tuple[str, str, str], ...] = ()
|
||||
if display_enabled:
|
||||
try:
|
||||
from deerflow.config.extensions_config import get_extensions_config
|
||||
|
||||
extensions = get_extensions_config()
|
||||
display_signature = tuple(
|
||||
(
|
||||
name,
|
||||
skill_config.display.mode,
|
||||
skill_config.display.custom_instruction,
|
||||
)
|
||||
for name, skill_config in sorted(extensions.skills.items())
|
||||
if skill_config.display is not None
|
||||
and skill_config.display.enabled
|
||||
and skill_config.display.mode != "none"
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to load skill display configs for prompt injection", exc_info=True)
|
||||
available_key = tuple(sorted(available_skills)) if available_skills is not None else None
|
||||
if not skill_signature and available_key is not None:
|
||||
return ""
|
||||
skill_evolution_section = _build_skill_evolution_section(effective_skill_evolution_enabled)
|
||||
return _get_cached_skills_prompt_section(skill_signature, available_key, container_base_path, skill_evolution_section)
|
||||
return _get_cached_skills_prompt_section(skill_signature, display_signature, available_key, container_base_path, skill_evolution_section)
|
||||
|
||||
|
||||
def get_agent_soul(agent_id: str | None) -> str:
|
||||
@@ -868,6 +914,7 @@ def apply_prompt_template(
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
) -> str:
|
||||
@@ -902,7 +949,11 @@ def apply_prompt_template(
|
||||
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
|
||||
|
||||
# Get skills section
|
||||
skills_section = get_skills_prompt_section(available_skills, app_config=app_config)
|
||||
skills_section = get_skills_prompt_section(
|
||||
available_skills,
|
||||
app_config=app_config,
|
||||
display_enabled=rag_mode_enabled,
|
||||
)
|
||||
|
||||
# Get deferred tools section (tool_search)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
@@ -940,6 +991,21 @@ Writing mode is enabled for this run.
|
||||
|
||||
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
|
||||
network_policy_section = _build_network_policy_section()
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- If the skill has a display adapter, follow that adapter first.
|
||||
- For `citation`, cite returned items with `[[1]]`, `[[2]]`, etc. Only cite real returned items.
|
||||
- For `table`, do not hand-write the table. Summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the table should appear. The frontend will render the real tool result as a stable table.
|
||||
- For `list` or `card`, summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="list" />` or `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the structured result should appear.
|
||||
- If no display adapter exists, default to citation mode with numbered citations.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
# Format the prompt with dynamic skills and memory
|
||||
prompt = SYSTEM_PROMPT_TEMPLATE.format(
|
||||
@@ -957,7 +1023,7 @@ Writing mode is enabled for this run.
|
||||
acp_section=acp_and_mounts_section,
|
||||
)
|
||||
|
||||
prompt = prompt + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
|
||||
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
|
||||
|
||||
if is_notebook_mode:
|
||||
source_ids = notebook_source_ids or []
|
||||
|
||||
+41
-7
@@ -61,6 +61,21 @@ _AUTH_PATTERNS = (
|
||||
"无权",
|
||||
"未授权",
|
||||
)
|
||||
# Upstream model gateway returned a malformed payload (typically a JSON decode
|
||||
# error wrapped as `bad_response_status_code`, or a truncated/garbled inference
|
||||
# response). These are almost always transient inference-side issues and safe
|
||||
# to retry.
|
||||
_UPSTREAM_GARBAGE_PATTERNS = (
|
||||
"bad_response_status_code",
|
||||
"bad response status",
|
||||
"expecting ',' delimiter",
|
||||
"expecting value",
|
||||
"expecting property name",
|
||||
"unexpected character",
|
||||
"unterminated string",
|
||||
"extra data",
|
||||
"jsondecodeerror",
|
||||
)
|
||||
|
||||
|
||||
class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
@@ -139,9 +154,10 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
detail = _extract_error_detail(exc)
|
||||
lowered = detail.lower()
|
||||
error_code = _extract_error_code(exc)
|
||||
error_code_lower = str(error_code).lower() if error_code is not None else ""
|
||||
status_code = _extract_status_code(exc)
|
||||
|
||||
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(str(error_code).lower(), _QUOTA_PATTERNS):
|
||||
if _matches_any(lowered, _QUOTA_PATTERNS) or _matches_any(error_code_lower, _QUOTA_PATTERNS):
|
||||
return False, "quota"
|
||||
if _matches_any(lowered, _AUTH_PATTERNS):
|
||||
return False, "auth"
|
||||
@@ -159,6 +175,11 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
return True, "transient"
|
||||
if _matches_any(lowered, _BUSY_PATTERNS):
|
||||
return True, "busy"
|
||||
# Treat upstream gateway garbage (malformed JSON, truncated responses)
|
||||
# as a retriable transient — the deterministic char-position errors we
|
||||
# see from vLLM/proxy layers usually clear on a re-roll.
|
||||
if _matches_any(lowered, _UPSTREAM_GARBAGE_PATTERNS) or _matches_any(error_code_lower, _UPSTREAM_GARBAGE_PATTERNS):
|
||||
return True, "upstream"
|
||||
|
||||
return False, "generic"
|
||||
|
||||
@@ -171,21 +192,34 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
|
||||
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
|
||||
seconds = max(1, round(wait_ms / 1000))
|
||||
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
|
||||
if reason == "busy":
|
||||
reason_text = "服务繁忙"
|
||||
elif reason == "upstream":
|
||||
reason_text = "上游响应异常"
|
||||
else:
|
||||
reason_text = "网络/连接异常"
|
||||
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
|
||||
|
||||
def _build_circuit_breaker_message(self) -> str:
|
||||
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
|
||||
|
||||
def _build_user_message(self, exc: BaseException, reason: str) -> str:
|
||||
detail = _extract_error_detail(exc)
|
||||
def _build_user_message(self, reason: str) -> str:
|
||||
if reason == "quota":
|
||||
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
|
||||
if reason == "auth":
|
||||
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
|
||||
if reason in {"busy", "transient"}:
|
||||
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
|
||||
return f"大模型因「{detail}」原因调用失败。"
|
||||
if reason == "upstream":
|
||||
return "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。"
|
||||
# NOTE: We intentionally do NOT embed the raw exception detail into the
|
||||
# user-facing message. Raw `Error code: 400 - {...}` strings — especially
|
||||
# Python repr-style dict literals — get persisted as AI messages and then
|
||||
# re-fed to the model on the next turn, where their unescaped quotes /
|
||||
# braces can corrupt the upstream JSON payload and cause deterministic
|
||||
# JSONDecodeError loops. Keep the user message clean; full detail is
|
||||
# already in the backend log via `logger.warning(..., exc_info=exc)`.
|
||||
return "大模型本次调用未能成功完成,请稍后再试或更换模型。"
|
||||
|
||||
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
|
||||
try:
|
||||
@@ -249,7 +283,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
)
|
||||
if retriable:
|
||||
self._record_failure()
|
||||
return AIMessage(content=self._build_user_message(exc, reason))
|
||||
return AIMessage(content=self._build_user_message(reason))
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
@@ -295,7 +329,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
)
|
||||
if retriable:
|
||||
self._record_failure()
|
||||
return AIMessage(content=self._build_user_message(exc, reason))
|
||||
return AIMessage(content=self._build_user_message(reason))
|
||||
|
||||
|
||||
def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool:
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""Inject stable citation numbering before RAG final answers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import SystemMessage
|
||||
|
||||
from deerflow.runtime.references import build_latest_reference_prompt, build_reference_debug_summary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
|
||||
|
||||
|
||||
def _get_rag_debug_logger() -> logging.Logger:
|
||||
debug_logger = logging.getLogger(_RAG_DEBUG_LOGGER_NAME)
|
||||
if getattr(debug_logger, "_rag_file_configured", False):
|
||||
return debug_logger
|
||||
log_path = Path(os.environ.get("RAG_CITATION_DEBUG_LOG", "logs/rag_citation_debug.log"))
|
||||
if not log_path.is_absolute():
|
||||
log_path = Path.cwd() / log_path
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.FileHandler(log_path, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
debug_logger.addHandler(handler)
|
||||
debug_logger.setLevel(logging.INFO)
|
||||
debug_logger.propagate = False
|
||||
setattr(debug_logger, "_rag_file_configured", True)
|
||||
setattr(debug_logger, "_rag_file_path", str(log_path))
|
||||
return debug_logger
|
||||
|
||||
|
||||
def _log_rag_debug(message: str, *args: Any) -> None:
|
||||
logger.info(message, *args)
|
||||
_get_rag_debug_logger().info(message, *args)
|
||||
|
||||
|
||||
class RagCitationMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Give the model the same numbered references the frontend will render."""
|
||||
|
||||
@staticmethod
|
||||
def _enabled(request: ModelRequest) -> bool:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
context = getattr(runtime, "context", None) if runtime is not None else None
|
||||
if isinstance(context, dict) and context.get("rag_mode_enabled"):
|
||||
return True
|
||||
config = getattr(runtime, "config", None) if runtime is not None else None
|
||||
if isinstance(config, dict):
|
||||
configurable = config.get("configurable")
|
||||
if isinstance(configurable, dict) and configurable.get("rag_mode_enabled"):
|
||||
return True
|
||||
runtime_context = config.get("context")
|
||||
if isinstance(runtime_context, dict) and runtime_context.get("rag_mode_enabled"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _patch(request: ModelRequest) -> ModelRequest | None:
|
||||
if not RagCitationMiddleware._enabled(request):
|
||||
return None
|
||||
messages = list(request.messages)
|
||||
prompt, batch = build_latest_reference_prompt(messages)
|
||||
if not prompt:
|
||||
_log_rag_debug(
|
||||
"middleware reference-prompt enabled=true prompt=false messages=%s",
|
||||
len(messages),
|
||||
)
|
||||
return None
|
||||
summary = build_reference_debug_summary(messages, [batch] if batch else None)
|
||||
_log_rag_debug(
|
||||
"middleware reference-prompt enabled=true prompt=true messages=%s tool_messages=%s "
|
||||
"raw_items=%s normalized_items=%s kept_items=%s latest_batch_sources=%s",
|
||||
summary["messages"],
|
||||
summary["tool_messages"],
|
||||
summary["raw_items"],
|
||||
summary["normalized_items"],
|
||||
summary["kept_items"],
|
||||
len((batch or {}).get("sources") or []),
|
||||
)
|
||||
patched_messages = [
|
||||
*messages,
|
||||
SystemMessage(content=prompt),
|
||||
]
|
||||
return request.override(messages=patched_messages)
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
patched = self._patch(request)
|
||||
return handler(patched or request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
patched = self._patch(request)
|
||||
return await handler(patched or request)
|
||||
@@ -840,10 +840,13 @@ class DeerFlowClient:
|
||||
|
||||
current_config = get_extensions_config()
|
||||
|
||||
config_data = {
|
||||
"mcpServers": mcp_servers,
|
||||
"skills": {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()},
|
||||
}
|
||||
config_data = {
|
||||
"mcpServers": mcp_servers,
|
||||
"skills": {
|
||||
name: skill.model_dump(by_alias=True, exclude_none=True)
|
||||
for name, skill in current_config.skills.items()
|
||||
},
|
||||
}
|
||||
|
||||
self._atomic_write_json(config_path, config_data)
|
||||
|
||||
@@ -903,13 +906,20 @@ class DeerFlowClient:
|
||||
if config_path is None:
|
||||
raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.")
|
||||
|
||||
extensions_config = get_extensions_config()
|
||||
extensions_config.skills[name] = SkillStateConfig(enabled=enabled)
|
||||
|
||||
config_data = {
|
||||
"mcpServers": {n: s.model_dump() for n, s in extensions_config.mcp_servers.items()},
|
||||
"skills": {n: {"enabled": sc.enabled} for n, sc in extensions_config.skills.items()},
|
||||
}
|
||||
extensions_config = get_extensions_config()
|
||||
current_skill_config = extensions_config.skills.get(name)
|
||||
extensions_config.skills[name] = SkillStateConfig(
|
||||
enabled=enabled,
|
||||
display=current_skill_config.display if current_skill_config else None,
|
||||
)
|
||||
|
||||
config_data = {
|
||||
"mcpServers": {n: s.model_dump(by_alias=True) for n, s in extensions_config.mcp_servers.items()},
|
||||
"skills": {
|
||||
n: sc.model_dump(by_alias=True, exclude_none=True)
|
||||
for n, sc in extensions_config.skills.items()
|
||||
},
|
||||
}
|
||||
|
||||
self._atomic_write_json(config_path, config_data)
|
||||
|
||||
|
||||
+58
-5
@@ -3,7 +3,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
@@ -53,10 +53,63 @@ class McpServerConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
"""Configuration for a single skill's state."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether this skill is enabled")
|
||||
class SkillDisplayTableColumn(BaseModel):
|
||||
"""A configured table column for structured skill result display."""
|
||||
|
||||
label: str = Field(description="Column label shown in the frontend table")
|
||||
field: str = Field(description="Field name or dot path read from each result item")
|
||||
width: str | None = Field(default=None, description="Optional CSS width for this column")
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayConfig(BaseModel):
|
||||
"""Frontend display adapter for structured skill results."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="none",
|
||||
description="How the frontend should render this skill's structured results",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="doubleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
"""Configuration for a single skill's state."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether this skill is enabled")
|
||||
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
|
||||
|
||||
|
||||
class ExtensionsConfig(BaseModel):
|
||||
|
||||
@@ -17,6 +17,7 @@ import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
@@ -37,10 +38,53 @@ class PromptPrefixSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class AppearanceSettings(BaseModel):
|
||||
"""System-wide appearance defaults that any *new* user inherits.
|
||||
|
||||
Once a user explicitly toggles their theme in the UI, that choice is
|
||||
persisted in the browser and supersedes this default — so changing
|
||||
``default_theme`` here only affects users who have never touched the
|
||||
toggle, and any future first-time loads.
|
||||
"""
|
||||
|
||||
default_theme: Literal["light", "dark"] = Field(
|
||||
default="light",
|
||||
description="Theme applied for users who haven't picked their own (light / dark)",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class CitationDisplaySettings(BaseModel):
|
||||
"""System-wide cleanup rules for citation cards and detail dialogs."""
|
||||
|
||||
cleanup_words: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Words or phrases used to remove noisy advertisement/disclaimer lines from citation display.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class PrivateSquareSettings(BaseModel):
|
||||
"""Admin-configurable settings for the agent private-square (私密广场).
|
||||
|
||||
The password gates read access to all agents flagged ``is_private=True``.
|
||||
The hash is stored opaquely — the router layer is responsible for hashing
|
||||
and verifying so the harness package keeps zero dependencies on
|
||||
``app.gateway.auth``.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the private square is offered to users at all")
|
||||
password_hash: str = Field(default="", description="Opaque hash of the admin-configured password. Empty = no password set yet.")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SystemSettings(BaseModel):
|
||||
"""Top-level system settings document."""
|
||||
|
||||
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
|
||||
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
|
||||
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
|
||||
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
@@ -81,3 +81,39 @@ class AgentStore(ABC):
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
"""Return every agent flagged ``is_private=True``.
|
||||
|
||||
Caller is responsible for authorization — typically the caller has
|
||||
already verified the admin-configured private-square password.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
"""Return every agent regardless of ownership/visibility.
|
||||
|
||||
Caller is responsible for authorization — typically the caller has
|
||||
already verified ``system_role == "admin"``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update any agent row by id, regardless of ownership.
|
||||
|
||||
Caller is responsible for authorization (admin only). Returns ``None``
|
||||
if the agent does not exist.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
"""Delete any agent row by id, regardless of ownership.
|
||||
|
||||
Caller is responsible for authorization (admin only). Returns
|
||||
``False`` if the agent does not exist.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+26
@@ -138,3 +138,29 @@ class MemoryAgentStore(AgentStore):
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
@@ -21,6 +21,12 @@ class AgentRow(Base):
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# When True, the agent is moved into the password-gated "private square"
|
||||
# (私密广场) and is hidden from the regular public square / built-in list.
|
||||
# The flag is independent of ``published``: a private agent is still
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
|
||||
+62
-6
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
@@ -28,10 +28,31 @@ class AgentRepository(AgentStore):
|
||||
|
||||
@staticmethod
|
||||
def _visible_filter(user_id: str):
|
||||
"""Visibility for single-record fetches (``get_visible``).
|
||||
|
||||
Lenient: a caller who already knows the agent id can fetch it even
|
||||
when it's flagged ``is_private``. The private-square password is the
|
||||
*discovery* gate (listings); individual reads are not re-gated, so a
|
||||
bookmarked / linked private agent stays openable.
|
||||
"""
|
||||
return or_(AgentRow.user_id.is_(None), AgentRow.user_id == user_id, AgentRow.published.is_(True))
|
||||
|
||||
@staticmethod
|
||||
def _listing_filter(user_id: str):
|
||||
"""Visibility for *listings* — hides private-square agents from
|
||||
non-owners so they only surface inside the password-gated endpoint.
|
||||
Owners keep seeing their own private agents in '我的'.
|
||||
"""
|
||||
return or_(
|
||||
AgentRow.user_id == user_id,
|
||||
and_(
|
||||
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
|
||||
AgentRow.is_private.is_(False),
|
||||
),
|
||||
)
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).where(self._visible_filter(user_id)).order_by(AgentRow.created_at.asc())
|
||||
stmt = select(AgentRow).where(self._listing_filter(user_id)).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
@@ -61,6 +82,7 @@ class AgentRepository(AgentStore):
|
||||
description=data.get("description") or "",
|
||||
user_id=data.get("user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
is_private=bool(data.get("is_private", False)),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -99,7 +121,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -112,7 +134,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -157,7 +179,7 @@ class AgentRepository(AgentStore):
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(AgentRow)
|
||||
.where(AgentRow.featured_order.is_not(None), self._visible_filter(user_id))
|
||||
.where(AgentRow.featured_order.is_not(None), self._listing_filter(user_id))
|
||||
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
@@ -168,9 +190,43 @@ class AgentRepository(AgentStore):
|
||||
if not agent_ids:
|
||||
return []
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._visible_filter(user_id))
|
||||
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._listing_filter(user_id))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars())
|
||||
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).where(AgentRow.is_private.is_(True)).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Add ``agents.is_private`` for the private-square (私密广场) feature.
|
||||
|
||||
A nullable-default bool column flagging agents that should live in the
|
||||
password-gated private square instead of the regular public square.
|
||||
|
||||
Revision ID: 20260526_01
|
||||
Revises: 20260522_01
|
||||
Create Date: 2026-05-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260526_01"
|
||||
down_revision = "20260522_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("agents") and not _has_column("agents", "is_private"):
|
||||
op.add_column(
|
||||
"agents",
|
||||
sa.Column(
|
||||
"is_private",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
if _has_table("agents") and not _has_index("agents", "ix_agents_is_private"):
|
||||
op.create_index("ix_agents_is_private", "agents", ["is_private"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_index("agents", "ix_agents_is_private"):
|
||||
op.drop_index("ix_agents_is_private", table_name="agents")
|
||||
if _has_column("agents", "is_private"):
|
||||
op.drop_column("agents", "is_private")
|
||||
+2
-1
@@ -30,7 +30,7 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
ScheduledTaskSubscriptionRow,
|
||||
SchedulerThreadRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
@@ -51,6 +51,7 @@ __all__ = [
|
||||
"ScheduledTaskSubscriptionRow",
|
||||
"SchedulerThreadRow",
|
||||
"SkillRow",
|
||||
"SkillDisplayAdapterRow",
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
|
||||
@@ -68,6 +68,14 @@ class SkillStore(ABC):
|
||||
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
"""Return every skill record regardless of ownership/visibility.
|
||||
|
||||
Caller is responsible for authorization (admin only).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_published_names(self) -> set[str]:
|
||||
"""Return the set of skill names with ``published = True`` (any owner).
|
||||
|
||||
+6
@@ -134,5 +134,11 @@ class MemorySkillStore(SkillStore):
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
_with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
for record in self._skills.values()
|
||||
]
|
||||
|
||||
async def list_published_names(self) -> set[str]:
|
||||
return {name for name, record in self._skills.items() if record.get("published")}
|
||||
+13
@@ -31,3 +31,16 @@ class SkillRow(Base):
|
||||
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class SkillDisplayAdapterRow(Base):
|
||||
__tablename__ = "skill_display_adapters"
|
||||
|
||||
skill_name: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Store JSON as text for maximum MySQL compatibility. Some deployments still
|
||||
# run MySQL variants where native JSON columns or unquoted names such as
|
||||
# ``display`` are not accepted consistently.
|
||||
display_config: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -149,6 +149,12 @@ class SkillRepository(SkillStore):
|
||||
async with self._sf() as session:
|
||||
return await session.scalar(stmt) or 0
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillRow).order_by(SkillRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_published_names(self) -> set[str]:
|
||||
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
|
||||
async with self._sf() as session:
|
||||
|
||||
@@ -0,0 +1,747 @@
|
||||
"""Build stable per-turn citation batches from LangGraph messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from deerflow.config.extensions_config import SkillDisplayConfig, get_extensions_config
|
||||
|
||||
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
|
||||
|
||||
def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
batches: list[dict[str, Any]] = []
|
||||
for index, message in enumerate(messages):
|
||||
if _message_type(message) != "human":
|
||||
continue
|
||||
end = len(messages)
|
||||
for cursor in range(index + 1, len(messages)):
|
||||
if _message_type(messages[cursor]) == "human":
|
||||
end = cursor
|
||||
break
|
||||
sources = _collect_sources(messages[index + 1 : end])
|
||||
if sources:
|
||||
anchor = _find_reference_batch_anchor(messages, index + 1, end)
|
||||
message_id = _message_id(anchor) or _message_id(message) or str(index)
|
||||
batches.append(
|
||||
{
|
||||
"id": message_id,
|
||||
"assistant_message_id": message_id,
|
||||
"sources": sources,
|
||||
}
|
||||
)
|
||||
return batches
|
||||
|
||||
|
||||
def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, Any] | None]:
|
||||
human_index = _find_latest_human_index(messages)
|
||||
if human_index is None:
|
||||
return "", None
|
||||
sources = _collect_sources(messages[human_index + 1 :])
|
||||
if not sources:
|
||||
return "", None
|
||||
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
|
||||
message_id = _message_id(assistant) or _message_id(messages[human_index]) or str(human_index)
|
||||
batch = {
|
||||
"id": message_id,
|
||||
"assistant_message_id": message_id,
|
||||
"sources": sources,
|
||||
}
|
||||
lines = [
|
||||
"本轮检索结果已由系统统一合并并重新编号。最终回答必须使用下面这套编号。",
|
||||
"如果你仍需要补充检索,请先继续调用工具;最终回答只能引用最新合并后的编号。",
|
||||
"",
|
||||
]
|
||||
for source in sources:
|
||||
title = source.get("title") or f"参考文献 {source.get('index')}"
|
||||
meta = " · ".join(
|
||||
str(item)
|
||||
for item in (source.get("source"), source.get("time"))
|
||||
if item
|
||||
)
|
||||
content = str(source.get("content") or source.get("snippet") or "").strip()
|
||||
if len(content) > 520:
|
||||
content = content[:517] + "..."
|
||||
head = f"[{source.get('index')}] {title}"
|
||||
if meta:
|
||||
head += f"({meta})"
|
||||
lines.append(head)
|
||||
if content:
|
||||
lines.append(content)
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"引用规则:",
|
||||
"1. 回答正文中的关键结论必须标注对应编号,例如 [1]、[2]。",
|
||||
"2. 多个来源可写为 [1][3][5]。",
|
||||
"3. 只能引用上方存在的编号,不要编造编号。",
|
||||
"4. 不要使用工具原始编号,也不要重新排序编号。",
|
||||
"5. 不要在正文末尾堆完整参考文献列表,前端会展示右侧参考文献。",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).strip(), batch
|
||||
|
||||
|
||||
def _find_latest_human_index(messages: list[Any]) -> int | None:
|
||||
for index in range(len(messages) - 1, -1, -1):
|
||||
if _message_type(messages[index]) == "human":
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def build_reference_debug_summary(messages: list[Any], batches: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
"""Return compact counts for citation extraction logs."""
|
||||
turn_summaries: list[dict[str, Any]] = []
|
||||
total_tool_messages = 0
|
||||
total_raw_items = 0
|
||||
total_normalized_items = 0
|
||||
total_kept_items = 0
|
||||
for index, message in enumerate(messages):
|
||||
if _message_type(message) != "human":
|
||||
continue
|
||||
end = len(messages)
|
||||
for cursor in range(index + 1, len(messages)):
|
||||
if _message_type(messages[cursor]) == "human":
|
||||
end = cursor
|
||||
break
|
||||
tool_messages = 0
|
||||
raw_items = 0
|
||||
normalized_items = 0
|
||||
kept_items = 0
|
||||
for tool_message in messages[index + 1 : end]:
|
||||
if _message_type(tool_message) != "tool":
|
||||
continue
|
||||
tool_messages += 1
|
||||
items = _first_list(_extract_tool_payload(tool_message))
|
||||
raw_items += len(items)
|
||||
for raw_item in items:
|
||||
normalized = _normalize_reference_item(raw_item, normalized_items + 1)
|
||||
if not normalized:
|
||||
continue
|
||||
normalized_items += 1
|
||||
kept_items += 1
|
||||
total_tool_messages += tool_messages
|
||||
total_raw_items += raw_items
|
||||
total_normalized_items += normalized_items
|
||||
total_kept_items += kept_items
|
||||
turn_summaries.append(
|
||||
{
|
||||
"human_index": index,
|
||||
"tool_messages": tool_messages,
|
||||
"raw_items": raw_items,
|
||||
"normalized_items": normalized_items,
|
||||
"kept_items": kept_items,
|
||||
}
|
||||
)
|
||||
batch_summaries = [
|
||||
{
|
||||
"id": batch.get("id"),
|
||||
"assistant_message_id": batch.get("assistant_message_id"),
|
||||
"source_count": len(batch.get("sources") or []),
|
||||
}
|
||||
for batch in (batches or build_reference_batches(messages))
|
||||
]
|
||||
return {
|
||||
"messages": len(messages),
|
||||
"turns": len(turn_summaries),
|
||||
"tool_messages": total_tool_messages,
|
||||
"raw_items": total_raw_items,
|
||||
"normalized_items": total_normalized_items,
|
||||
"kept_items": total_kept_items,
|
||||
"turns_detail": turn_summaries,
|
||||
"batches": batch_summaries,
|
||||
}
|
||||
|
||||
|
||||
def adapt_reference_batches_with_displays(
|
||||
batches: list[dict[str, Any]],
|
||||
displays: dict[str, SkillDisplayConfig | dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Apply persisted skill display configs to already-built reference batches.
|
||||
|
||||
The reference builder can run before the gateway has a fresh extensions
|
||||
cache, while the skill adapter editor persists the user's latest mapping in
|
||||
the database. This pass lets the API response use that DB-backed mapping
|
||||
without changing citation indexes.
|
||||
"""
|
||||
normalized_displays: dict[str, SkillDisplayConfig] = {}
|
||||
for skill_name, display in displays.items():
|
||||
try:
|
||||
config = (
|
||||
display
|
||||
if isinstance(display, SkillDisplayConfig)
|
||||
else SkillDisplayConfig.model_validate(display)
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
if not config.enabled or config.mode == "none":
|
||||
continue
|
||||
normalized_displays[skill_name] = config
|
||||
|
||||
if not normalized_displays:
|
||||
return batches
|
||||
|
||||
displays_by_normalized_name = {
|
||||
_normalize_name(name): (name, display)
|
||||
for name, display in normalized_displays.items()
|
||||
}
|
||||
adapted_batches: list[dict[str, Any]] = []
|
||||
for batch in batches:
|
||||
sources: list[dict[str, Any]] = []
|
||||
for source in batch.get("sources", []):
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
skill_name = str(source.get("skillName") or "")
|
||||
display = normalized_displays.get(skill_name)
|
||||
if display is None:
|
||||
normalized_source = _normalize_name(skill_name)
|
||||
normalized_match = displays_by_normalized_name.get(normalized_source)
|
||||
if normalized_match is None:
|
||||
for normalized_skill, candidate in displays_by_normalized_name.items():
|
||||
if (
|
||||
normalized_source.endswith(normalized_skill)
|
||||
or normalized_skill in normalized_source
|
||||
):
|
||||
normalized_match = candidate
|
||||
break
|
||||
if normalized_match is not None:
|
||||
skill_name, display = normalized_match
|
||||
raw = source.get("raw")
|
||||
if display is not None and isinstance(raw, dict):
|
||||
index_value = int(source.get("index") or len(sources) + 1)
|
||||
configured = _normalize_with_display_config(
|
||||
raw,
|
||||
index_value,
|
||||
skill_name,
|
||||
display,
|
||||
)
|
||||
if configured is not None:
|
||||
sources.append(
|
||||
{
|
||||
**source,
|
||||
**configured,
|
||||
"index": index_value,
|
||||
"skillName": skill_name,
|
||||
"raw": raw,
|
||||
}
|
||||
)
|
||||
continue
|
||||
sources.append(source)
|
||||
adapted_batches.append({**batch, "sources": sources})
|
||||
return adapted_batches
|
||||
|
||||
|
||||
def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
sources: list[dict[str, Any]] = []
|
||||
pending_tool_contexts: list[dict[str, Any]] = []
|
||||
tool_contexts_by_id: dict[str, dict[str, Any]] = {}
|
||||
for message in messages:
|
||||
message_type = _message_type(message)
|
||||
if message_type == "ai":
|
||||
for tool_call in _extract_tool_calls(message):
|
||||
context = _build_tool_context(tool_call)
|
||||
pending_tool_contexts.append(context)
|
||||
if context.get("id"):
|
||||
tool_contexts_by_id[str(context["id"])] = context
|
||||
continue
|
||||
if message_type != "tool":
|
||||
continue
|
||||
tool_context = _resolve_tool_context(
|
||||
message,
|
||||
pending_tool_contexts,
|
||||
tool_contexts_by_id,
|
||||
)
|
||||
payload = _extract_tool_payload(message)
|
||||
for raw_item in _first_list(payload):
|
||||
normalized = _normalize_reference_item(
|
||||
raw_item,
|
||||
len(sources) + 1,
|
||||
payload=payload,
|
||||
tool_context=tool_context,
|
||||
)
|
||||
if not normalized:
|
||||
continue
|
||||
normalized["index"] = len(sources) + 1
|
||||
sources.append(normalized)
|
||||
return sources
|
||||
|
||||
|
||||
def _find_reference_batch_anchor(messages: list[Any], start: int, end: int) -> Any | None:
|
||||
answer = _find_last_assistant_answer(messages, start, end)
|
||||
if answer is not None:
|
||||
return answer
|
||||
for index in range(end - 1, start - 1, -1):
|
||||
candidate = messages[index]
|
||||
if _message_type(candidate) == "ai" and _message_id(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _find_last_assistant_answer(messages: list[Any], start: int, end: int) -> Any | None:
|
||||
for index in range(end - 1, start - 1, -1):
|
||||
candidate = messages[index]
|
||||
if (
|
||||
_message_type(candidate) == "ai"
|
||||
and _message_id(candidate)
|
||||
and _has_answer_content(candidate)
|
||||
):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _message_type(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("type") or message.get("role") or "")
|
||||
return str(getattr(message, "type", "") or "")
|
||||
|
||||
|
||||
def _message_id(message: Any) -> str | None:
|
||||
value = message.get("id") if isinstance(message, dict) else getattr(message, "id", None)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _message_content(message: Any) -> Any:
|
||||
return message.get("content") if isinstance(message, dict) else getattr(message, "content", None)
|
||||
|
||||
|
||||
def _message_name(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
|
||||
return str(getattr(message, "name", None) or "tool")
|
||||
|
||||
|
||||
def _message_tool_call_id(message: Any) -> str | None:
|
||||
if isinstance(message, dict):
|
||||
value = message.get("tool_call_id") or message.get("toolCallId")
|
||||
else:
|
||||
value = getattr(message, "tool_call_id", None) or getattr(message, "toolCallId", None)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _has_tool_calls(message: Any) -> bool:
|
||||
if isinstance(message, dict):
|
||||
direct = message.get("tool_calls")
|
||||
chunks = message.get("tool_call_chunks")
|
||||
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
|
||||
else:
|
||||
direct = getattr(message, "tool_calls", None)
|
||||
chunks = getattr(message, "tool_call_chunks", None)
|
||||
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
|
||||
return bool(direct or chunks or extra)
|
||||
|
||||
|
||||
def _has_answer_content(message: Any) -> bool:
|
||||
content = _message_content(message)
|
||||
if isinstance(content, str):
|
||||
return bool(content.strip())
|
||||
if isinstance(content, list):
|
||||
return any(
|
||||
isinstance(part, dict)
|
||||
and part.get("type") == "text"
|
||||
and str(part.get("text") or "").strip()
|
||||
for part in content
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _extract_tool_calls(message: Any) -> list[dict[str, Any]]:
|
||||
candidates: list[Any] = []
|
||||
if isinstance(message, dict):
|
||||
for key in ("tool_calls", "tool_call_chunks"):
|
||||
value = message.get(key)
|
||||
if isinstance(value, list):
|
||||
candidates.extend(value)
|
||||
extra = (message.get("additional_kwargs") or {}).get("tool_calls")
|
||||
if isinstance(extra, list):
|
||||
candidates.extend(extra)
|
||||
else:
|
||||
for key in ("tool_calls", "tool_call_chunks"):
|
||||
value = getattr(message, key, None)
|
||||
if isinstance(value, list):
|
||||
candidates.extend(value)
|
||||
extra = (getattr(message, "additional_kwargs", None) or {}).get("tool_calls")
|
||||
if isinstance(extra, list):
|
||||
candidates.extend(extra)
|
||||
|
||||
calls: list[dict[str, Any]] = []
|
||||
seen_ids: set[str] = set()
|
||||
for raw in candidates:
|
||||
call = _normalize_tool_call(raw)
|
||||
if not call:
|
||||
continue
|
||||
call_id = str(call.get("id") or "")
|
||||
if call_id and call_id in seen_ids:
|
||||
continue
|
||||
if call_id:
|
||||
seen_ids.add(call_id)
|
||||
calls.append(call)
|
||||
return calls
|
||||
|
||||
|
||||
def _normalize_tool_call(raw: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
fn = raw.get("function") if isinstance(raw.get("function"), dict) else {}
|
||||
name = raw.get("name") or fn.get("name") or raw.get("tool_name")
|
||||
if not name:
|
||||
return None
|
||||
return {
|
||||
"id": raw.get("id") or raw.get("tool_call_id") or raw.get("call_id"),
|
||||
"name": str(name),
|
||||
"args": _normalize_tool_args(raw.get("args") or raw.get("arguments") or fn.get("arguments")),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_tool_args(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
pass
|
||||
return {"command": value}
|
||||
return {}
|
||||
|
||||
|
||||
def _build_tool_context(tool_call: dict[str, Any]) -> dict[str, Any]:
|
||||
tool_name = str(tool_call.get("name") or "tool")
|
||||
args = tool_call.get("args") if isinstance(tool_call.get("args"), dict) else {}
|
||||
return {
|
||||
"id": tool_call.get("id"),
|
||||
"tool_name": tool_name,
|
||||
"args": args,
|
||||
"skill_name": _infer_skill_name(tool_name, args),
|
||||
}
|
||||
|
||||
|
||||
def _resolve_tool_context(
|
||||
message: Any,
|
||||
pending_tool_contexts: list[dict[str, Any]],
|
||||
tool_contexts_by_id: dict[str, dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
tool_call_id = _message_tool_call_id(message)
|
||||
if tool_call_id and tool_call_id in tool_contexts_by_id:
|
||||
context = tool_contexts_by_id[tool_call_id]
|
||||
pending_tool_contexts[:] = [
|
||||
item for item in pending_tool_contexts if str(item.get("id") or "") != tool_call_id
|
||||
]
|
||||
return context
|
||||
if pending_tool_contexts:
|
||||
return pending_tool_contexts.pop(0)
|
||||
tool_name = _message_name(message)
|
||||
return {
|
||||
"id": tool_call_id,
|
||||
"tool_name": tool_name,
|
||||
"args": {},
|
||||
"skill_name": _infer_skill_name(tool_name, {}),
|
||||
}
|
||||
|
||||
|
||||
def _extract_tool_payload(message: Any) -> Any:
|
||||
content = _message_content(message)
|
||||
if isinstance(content, dict):
|
||||
return _unwrap_tool_payload(content)
|
||||
if isinstance(content, list):
|
||||
texts: list[str] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if isinstance(part.get("json"), dict):
|
||||
return _unwrap_tool_payload(part["json"])
|
||||
if part.get("type") == "json" and isinstance(part.get("json"), dict):
|
||||
return _unwrap_tool_payload(part["json"])
|
||||
if part.get("type") == "text" and isinstance(part.get("text"), str):
|
||||
texts.append(part["text"])
|
||||
unwrapped = _unwrap_tool_payload(part)
|
||||
if unwrapped is not None:
|
||||
return unwrapped
|
||||
content = "\n".join(texts)
|
||||
if isinstance(content, str):
|
||||
return _parse_json_like(content)
|
||||
return None
|
||||
|
||||
|
||||
def _unwrap_tool_payload(value: dict[str, Any]) -> Any:
|
||||
list_keys = ("results", "data", "items", "hits", "documents", "records", "sources")
|
||||
if any(isinstance(value.get(key), list) for key in list_keys):
|
||||
return value
|
||||
for key in ("result", "output", "stdout", "content", "text", "payload", "response"):
|
||||
nested = value.get(key)
|
||||
if isinstance(nested, str):
|
||||
parsed = _parse_json_like(nested)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
if isinstance(nested, dict):
|
||||
unwrapped = _unwrap_tool_payload(nested)
|
||||
if unwrapped is not None:
|
||||
return unwrapped
|
||||
return value
|
||||
|
||||
|
||||
def _parse_json_like(text: str) -> Any:
|
||||
trimmed = text.strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
try:
|
||||
return json.loads(trimmed)
|
||||
except Exception:
|
||||
pass
|
||||
first = trimmed.find("{")
|
||||
last = trimmed.rfind("}")
|
||||
if 0 <= first < last:
|
||||
try:
|
||||
return json.loads(trimmed[first : last + 1])
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _first_list(payload: Any) -> list[Any]:
|
||||
if isinstance(payload, list):
|
||||
return payload
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
for key in ("results", "data", "items", "hits", "documents", "records", "entries", "matches", "chunks", "sources"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
for key in ("result", "payload", "response", "output", "data"):
|
||||
found = _first_list(payload.get(key))
|
||||
if found:
|
||||
return found
|
||||
return []
|
||||
|
||||
|
||||
def _nested_records(raw: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(raw, dict):
|
||||
return []
|
||||
layers = [raw]
|
||||
for key in ("metadata", "meta", "attrs", "attributes", "properties", "extra"):
|
||||
nested = raw.get(key)
|
||||
if isinstance(nested, dict):
|
||||
layers.append(nested)
|
||||
return layers
|
||||
|
||||
|
||||
def _pick(layers: list[dict[str, Any]], keys: tuple[str, ...]) -> str | None:
|
||||
for layer in layers:
|
||||
for key in keys:
|
||||
value = layer.get(key)
|
||||
if isinstance(value, (str, int, float)) and str(value).strip():
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _enabled_skill_displays() -> dict[str, SkillDisplayConfig]:
|
||||
try:
|
||||
return {
|
||||
name: state.display
|
||||
for name, state in get_extensions_config().skills.items()
|
||||
if state.display is not None
|
||||
and state.enabled
|
||||
and state.display.enabled
|
||||
and state.display.mode != "none"
|
||||
}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _infer_skill_name(tool_name: str, args: dict[str, Any]) -> str | None:
|
||||
displays = _enabled_skill_displays()
|
||||
if tool_name in displays:
|
||||
return tool_name
|
||||
normalized_tool = _normalize_name(tool_name)
|
||||
for skill_name in displays:
|
||||
normalized_skill = _normalize_name(skill_name)
|
||||
if (
|
||||
normalized_tool == normalized_skill
|
||||
or normalized_tool.endswith(normalized_skill)
|
||||
or normalized_skill in normalized_tool
|
||||
):
|
||||
return skill_name
|
||||
for value in args.values():
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
match = _SKILL_PATH_RE.search(value)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_display_match(
|
||||
raw: dict[str, Any],
|
||||
payload: Any,
|
||||
tool_context: dict[str, Any] | None,
|
||||
) -> tuple[str, SkillDisplayConfig] | None:
|
||||
displays = _enabled_skill_displays()
|
||||
if not displays:
|
||||
return None
|
||||
skill_name = str((tool_context or {}).get("skill_name") or "")
|
||||
display = displays.get(skill_name)
|
||||
if display is not None:
|
||||
return skill_name, display
|
||||
tool_name = str((tool_context or {}).get("tool_name") or "")
|
||||
inferred = _infer_skill_name(tool_name, (tool_context or {}).get("args") or {})
|
||||
if inferred and inferred in displays:
|
||||
return inferred, displays[inferred]
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_with_display_config(
|
||||
raw: dict[str, Any],
|
||||
fallback_index: int,
|
||||
skill_name: str,
|
||||
display: SkillDisplayConfig,
|
||||
) -> dict[str, Any] | None:
|
||||
title = _stringify(_get_by_path(raw, display.title_field), f"结果 {fallback_index}")
|
||||
snippet = _stringify(_get_by_path(raw, display.summary_field), "")
|
||||
content = _stringify(_get_by_path(raw, display.content_field), snippet)
|
||||
source = _stringify(_get_by_path(raw, display.source_field), "") or None
|
||||
item_type = _stringify(_get_by_path(raw, display.type_field), "result") or "result"
|
||||
time = _stringify(_get_by_path(raw, display.time_field), "") or None
|
||||
item_id = _stringify(_get_by_path(raw, display.id_field), "") or None
|
||||
score = _number_or_none(_get_by_path(raw, display.score_field))
|
||||
url = _resolve_display_link(raw, display)
|
||||
if not title and not content and not url:
|
||||
return None
|
||||
return {
|
||||
"id": item_id or url,
|
||||
"skillName": skill_name,
|
||||
"mode": display.mode,
|
||||
"title": title,
|
||||
"type": item_type,
|
||||
"snippet": snippet,
|
||||
"content": content,
|
||||
"source": source,
|
||||
"time": time,
|
||||
"url": url,
|
||||
"score": score,
|
||||
"tableColumns": [column.model_dump(by_alias=True) for column in display.table_columns],
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_display_link(raw: dict[str, Any], display: SkillDisplayConfig) -> str | None:
|
||||
url = _stringify(_get_by_path(raw, display.url_field), "")
|
||||
item_id = _stringify(_get_by_path(raw, display.id_field), "")
|
||||
if display.link_strategy == "url":
|
||||
return url or None
|
||||
if display.link_strategy == "id":
|
||||
return _render_template(display.link_template, raw) if item_id and display.link_template else None
|
||||
return url or (_render_template(display.link_template, raw) if item_id and display.link_template else None) or None
|
||||
|
||||
|
||||
def _render_template(template: str | None, raw: dict[str, Any]) -> str | None:
|
||||
if not template:
|
||||
return None
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
value = _get_by_path(raw, match.group(1).strip())
|
||||
return quote(_stringify(value, ""), safe="")
|
||||
|
||||
return re.sub(r"\{([^}]+)\}", replace, template)
|
||||
|
||||
|
||||
def _get_by_path(value: Any, path: str | None) -> Any:
|
||||
if not path:
|
||||
return None
|
||||
cursor = value
|
||||
for part in path.split("."):
|
||||
if isinstance(cursor, dict):
|
||||
cursor = cursor.get(part)
|
||||
else:
|
||||
return None
|
||||
return cursor
|
||||
|
||||
|
||||
def _stringify(value: Any, fallback: str = "") -> str:
|
||||
if value is None:
|
||||
return fallback
|
||||
if isinstance(value, (str, int, float)):
|
||||
text = str(value).strip()
|
||||
return text if text else fallback
|
||||
return fallback
|
||||
|
||||
|
||||
def _number_or_none(value: Any) -> float | None:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_name(name: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", name.lower())
|
||||
|
||||
|
||||
def _normalize_reference_item(
|
||||
raw: Any,
|
||||
fallback_index: int,
|
||||
*,
|
||||
payload: Any = None,
|
||||
tool_context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
display_match = _resolve_display_match(raw, payload, tool_context)
|
||||
if display_match is not None:
|
||||
skill_name, display = display_match
|
||||
configured = _normalize_with_display_config(raw, fallback_index, skill_name, display)
|
||||
if configured is not None:
|
||||
return configured
|
||||
layers = _nested_records(raw)
|
||||
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
|
||||
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
|
||||
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
|
||||
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
|
||||
time = _pick(layers, ("time", "date", "published_date", "publishedDate", "m_publish", "publish_time", "created_at"))
|
||||
item_id = _pick(layers, ("recUuid", "rec_uuid", "uuid", "docId", "doc_id", "id", "recordId", "taskId"))
|
||||
score_raw = _pick(layers, ("score", "relevance", "similarity"))
|
||||
score = None
|
||||
if score_raw is not None:
|
||||
try:
|
||||
score = float(score_raw)
|
||||
except Exception:
|
||||
score = None
|
||||
if not title:
|
||||
title = source or f"结果 {fallback_index}"
|
||||
if not content:
|
||||
content = ""
|
||||
if not title and not content and not url:
|
||||
return None
|
||||
return {
|
||||
"id": item_id or url,
|
||||
"skillName": str((tool_context or {}).get("skill_name") or (tool_context or {}).get("tool_name") or "backend-reference"),
|
||||
"mode": "citation",
|
||||
"title": title,
|
||||
"type": "result",
|
||||
"snippet": content,
|
||||
"content": content,
|
||||
"source": source,
|
||||
"time": time,
|
||||
"url": url,
|
||||
"score": score,
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _reference_key(source: dict[str, Any]) -> str:
|
||||
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
|
||||
item_id = source.get("id") or source.get("url") or raw.get("recUuid") or raw.get("uuid") or raw.get("docId") or raw.get("id")
|
||||
if item_id:
|
||||
return str(item_id).casefold().strip()
|
||||
return "|".join(
|
||||
str(source.get(key) or "").casefold().strip()
|
||||
for key in ("title", "source", "time", "content", "snippet")
|
||||
)
|
||||
+48
-27
@@ -37,14 +37,26 @@ class LocalSandbox(Sandbox):
|
||||
"""Return whether the selected shell is a PowerShell executable."""
|
||||
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _is_cmd_shell(shell: str) -> bool:
|
||||
"""Return whether the selected shell is cmd.exe."""
|
||||
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
|
||||
"""Return the first executable shell path or command found from candidates."""
|
||||
@staticmethod
|
||||
def _is_cmd_shell(shell: str) -> bool:
|
||||
"""Return whether the selected shell is cmd.exe."""
|
||||
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_powershell_command(command: str) -> str:
|
||||
"""Make bash-style command glue usable in Windows PowerShell 5.
|
||||
|
||||
The public tool is still named ``bash`` for model compatibility, but on
|
||||
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
|
||||
not support ``&&``, while the sandbox layer prepends commands with
|
||||
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
|
||||
executable in that fallback shell.
|
||||
"""
|
||||
return command.replace("&&", ";")
|
||||
|
||||
@staticmethod
|
||||
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
|
||||
"""Return the first executable shell path or command found from candidates."""
|
||||
for shell in candidates:
|
||||
if os.path.isabs(shell):
|
||||
if os.path.isfile(shell) and os.access(shell, os.X_OK):
|
||||
@@ -302,30 +314,39 @@ class LocalSandbox(Sandbox):
|
||||
resolved_command = self._resolve_paths_in_command(command)
|
||||
shell = self._get_shell()
|
||||
|
||||
if os.name == "nt":
|
||||
if self._is_powershell(shell):
|
||||
args = [shell, "-NoProfile", "-Command", resolved_command]
|
||||
if os.name == "nt":
|
||||
if self._is_powershell(shell):
|
||||
args = [
|
||||
shell,
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
self._normalize_powershell_command(resolved_command),
|
||||
]
|
||||
elif self._is_cmd_shell(shell):
|
||||
args = [shell, "/c", resolved_command]
|
||||
else:
|
||||
args = [shell, "-c", resolved_command]
|
||||
|
||||
result = subprocess.run(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
args = [shell, "-c", resolved_command]
|
||||
result = subprocess.run(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
result = subprocess.run(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
args = [shell, "-c", resolved_command]
|
||||
result = subprocess.run(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=600,
|
||||
)
|
||||
output = result.stdout
|
||||
if result.stderr:
|
||||
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
|
||||
|
||||
@@ -149,3 +149,58 @@ async def cascade_skill_unpublished_or_deleted(
|
||||
counters["notified"] += 1
|
||||
|
||||
return counters
|
||||
|
||||
|
||||
def sync_agent_skills_to_available(available_skill_names: set[str]) -> dict[str, Any]:
|
||||
"""Strip skill references that no longer exist on disk from every agent.
|
||||
|
||||
Walks every agent dir and rewrites its ``config.yaml`` ``skills`` list to
|
||||
contain only names that appear in ``available_skill_names``. Used by the
|
||||
admin "agent-skills sync" endpoint to repair historical drift — skills
|
||||
deleted before the cascade existed, deleted via a path that skipped the
|
||||
cascade, or removed from the filesystem manually.
|
||||
|
||||
Synchronous because it only touches local files and the admin endpoint
|
||||
runs it in a worker thread.
|
||||
|
||||
Returns ``{checked, modified, removals: [{agent_id, removed_skills}]}``.
|
||||
"""
|
||||
counters: dict[str, Any] = {"checked": 0, "modified": 0, "removals": []}
|
||||
|
||||
for agent_dir in _list_agent_dirs():
|
||||
config_file = agent_dir / "config.yaml"
|
||||
if not config_file.exists():
|
||||
continue
|
||||
counters["checked"] += 1
|
||||
|
||||
try:
|
||||
with config_file.open("r", encoding="utf-8") as f:
|
||||
data: dict[str, Any] = yaml.safe_load(f) or {}
|
||||
except yaml.YAMLError:
|
||||
logger.warning("Sync: failed to parse %s, skipping", config_file)
|
||||
continue
|
||||
|
||||
skills = data.get("skills")
|
||||
if not isinstance(skills, list) or not skills:
|
||||
continue
|
||||
|
||||
kept: list[str] = []
|
||||
removed: list[str] = []
|
||||
for name in skills:
|
||||
if isinstance(name, str) and name in available_skill_names:
|
||||
kept.append(name)
|
||||
else:
|
||||
removed.append(name if isinstance(name, str) else str(name))
|
||||
|
||||
if not removed:
|
||||
continue
|
||||
|
||||
data["skills"] = kept
|
||||
with config_file.open("w", encoding="utf-8") as f:
|
||||
yaml.safe_dump(data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
counters["modified"] += 1
|
||||
counters["removals"].append({"agent_id": agent_dir.name, "removed_skills": removed})
|
||||
logger.info("Sync: removed orphan skills %s from agent '%s'", removed, agent_dir.name)
|
||||
|
||||
return counters
|
||||
Reference in new issue
Block a user