sync update from LAN repo (source dhk-1 @ c21a2a4a)

This commit is contained in:
gozeros committed 2026-11-10 14:04:00 +08:00
1 parent 264e2200d6
commit a1fa13a238
122 files changed
+8881 -623

No files matched your search

@@ -264,17 +264,17 @@ def _get_enabled_skills():
def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[Skill]:
"""Return enabled skills using the caller's config source.
Resolve storage from the caller's config and reuse the parsed catalogue only
when its root matches. A cold cache or a different root falls back to that
storage, keeping request-scoped skill directories isolated.
Resolve storage from the caller's config and reuse the parsed catalogue only
when its root matches. A cold cache or a different root falls back to that
storage, keeping request-scoped skill directories isolated.
"""
if app_config is None:
return _get_enabled_skills()
storage = get_or_new_skill_storage(app_config=app_config)
warm = get_warm_enabled_skills(storage)
if warm is not None:
return warm
return _apply_always_on(_drop_archived_skills(list(storage.load_skills(enabled_only=True))))
storage = get_or_new_skill_storage(app_config=app_config)
warm = get_warm_enabled_skills(storage)
if warm is not None:
return warm
return _apply_always_on(_drop_archived_skills(list(storage.load_skills(enabled_only=True))))
def _skill_mutability_label(category: SkillCategory | str) -> str:
@@ -617,44 +617,44 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p
</subagent_system>"""
_ORDINARY_QA_HEADING_RULES = {
"hybrid_decimal": """- 一级章节使用 `## 一、标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
- 二级章节按 `1.1`、`1.2`、`2.1` 的形式编号,三级及更深层级继续使用完整多级阿拉伯数字。
- 确需更深层级时,四级章节使用 `##### 1.1.1.1 标题`,五级章节使用 `###### 1.1.1.1.1 标题`。""",
"official_chinese": """- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。
- 确需更深层级时,四级章节使用 `##### (1) 标题`,五级章节使用 `###### 1) 标题`。""",
"decimal": """- 一级章节使用 `## 1 标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
- 确需更深层级时,继续使用 `##### 1.1.1.1 标题`、`###### 1.1.1.1.1 标题`。""",
"chapter_section": """- 一级章节使用 `## 第一章 标题`,二级章节使用 `### 第一节 标题`,三级章节使用 `#### 一、标题`。
- 确需更深层级时,四级章节使用 `##### (一) 标题`,五级章节使用 `###### 1. 标题`。""",
"none": """- 各级章节只写标题文字,不手写编号:`## 一级标题`、`### 二级标题`、`#### 三级标题`。
- 确需更深层级时使用 `##### 四级标题`、`###### 五级标题`,仍不添加编号。""",
}
def _word_heading_numbering_style() -> str:
"""Return the configured Word heading-numbering preset with a safe default."""
try:
return load_system_settings().prompt_prefix.word_heading_numbering_style
except Exception:
logger.debug("Failed to read the Word heading-numbering style; using default", exc_info=True)
return "hybrid_decimal"
def _build_ordinary_qa_markdown_format_section() -> str:
rules = _ORDINARY_QA_HEADING_RULES.get(
_word_heading_numbering_style(),
_ORDINARY_QA_HEADING_RULES["hybrid_decimal"],
)
return f"""
<ordinary_qa_markdown_format>
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
{rules}
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
</ordinary_qa_markdown_format>
"""
_ORDINARY_QA_HEADING_RULES = {
"hybrid_decimal": """- 一级章节使用 `## 一、标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
- 二级章节按 `1.1`、`1.2`、`2.1` 的形式编号,三级及更深层级继续使用完整多级阿拉伯数字。
- 确需更深层级时,四级章节使用 `##### 1.1.1.1 标题`,五级章节使用 `###### 1.1.1.1.1 标题`。""",
"official_chinese": """- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。
- 确需更深层级时,四级章节使用 `##### (1) 标题`,五级章节使用 `###### 1) 标题`。""",
"decimal": """- 一级章节使用 `## 1 标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
- 确需更深层级时,继续使用 `##### 1.1.1.1 标题`、`###### 1.1.1.1.1 标题`。""",
"chapter_section": """- 一级章节使用 `## 第一章 标题`,二级章节使用 `### 第一节 标题`,三级章节使用 `#### 一、标题`。
- 确需更深层级时,四级章节使用 `##### (一) 标题`,五级章节使用 `###### 1. 标题`。""",
"none": """- 各级章节只写标题文字,不手写编号:`## 一级标题`、`### 二级标题`、`#### 三级标题`。
- 确需更深层级时使用 `##### 四级标题`、`###### 五级标题`,仍不添加编号。""",
}
def _word_heading_numbering_style() -> str:
"""Return the configured Word heading-numbering preset with a safe default."""
try:
return load_system_settings().prompt_prefix.word_heading_numbering_style
except Exception:
logger.debug("Failed to read the Word heading-numbering style; using default", exc_info=True)
return "hybrid_decimal"
def _build_ordinary_qa_markdown_format_section() -> str:
rules = _ORDINARY_QA_HEADING_RULES.get(
_word_heading_numbering_style(),
_ORDINARY_QA_HEADING_RULES["hybrid_decimal"],
)
return f"""
<ordinary_qa_markdown_format>
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
{rules}
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
</ordinary_qa_markdown_format>
"""
_ORDINARY_QA_VISUALIZATION_SECTION = """
@@ -1261,13 +1261,13 @@ RAG/reference display mode is enabled.
return prompt
def _get_memory_context(
agent_name: str | None = None,
*,
app_config: AppConfig | None = None,
injection_enabled: bool = True,
include_legacy_knowledge_routing: bool = True,
) -> str:
def _get_memory_context(
agent_name: str | None = None,
*,
app_config: AppConfig | None = None,
injection_enabled: bool = True,
include_legacy_knowledge_routing: bool = True,
) -> str:
"""组装注入系统提示词的记忆上下文 (Memory V2)。
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
@@ -1278,10 +1278,10 @@ def _get_memory_context(
agent_name: agent 标识;``None`` 时按 "default" 处理。
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
injection_enabled: 运行时开关;``False`` 时跳过注入直接返回空串。
include_legacy_knowledge_routing: 是否注入 ``/mnt/knowledge/routing.md``。
智能普通知识库检索已有数据库授权范围和专用检索工具时必须关闭,
避免旧文件路由把“这个库”错误引向沙箱目录;其它记忆 provider
仍按原配置保留。
include_legacy_knowledge_routing: 是否注入 ``/mnt/knowledge/routing.md``。
智能普通知识库检索已有数据库授权范围和专用检索工具时必须关闭,
避免旧文件路由把“这个库”错误引向沙箱目录;其它记忆 provider
仍按原配置保留。
Returns:
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
@@ -1306,11 +1306,11 @@ def _get_memory_context(
)
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
context_content = manager.build_system_prompt(
excluded_provider_names=(
None if include_legacy_knowledge_routing else {"knowledge"}
)
)
context_content = manager.build_system_prompt(
excluded_provider_names=(
None if include_legacy_knowledge_routing else {"knowledge"}
)
)
if not context_content.strip():
return ""
@@ -1878,7 +1878,7 @@ def apply_prompt_template(
is_bootstrap: bool = False,
app_config: AppConfig | None = None,
memory_injection_enabled: bool = True,
include_legacy_knowledge_routing: bool = True,
include_legacy_knowledge_routing: bool = True,
rag_mode_enabled: bool = False,
is_notebook_mode: bool = False,
notebook_source_ids: list[int] | None = None,
@@ -1887,12 +1887,12 @@ def apply_prompt_template(
storage_agent_id = agent_id if agent_id is not None else agent_name
# Get memory context
memory_context = _get_memory_context(
storage_agent_id,
app_config=app_config,
injection_enabled=memory_injection_enabled,
include_legacy_knowledge_routing=include_legacy_knowledge_routing,
)
memory_context = _get_memory_context(
storage_agent_id,
app_config=app_config,
injection_enabled=memory_injection_enabled,
include_legacy_knowledge_routing=include_legacy_knowledge_routing,
)
# Include subagent section only if enabled (from runtime parameter)
n = max_concurrent_subagents
@@ -2004,8 +2004,8 @@ When a search, knowledge, skill, or MCP tool returns a result list:
if rag_mode_enabled
else ""
)
ordinary_qa_format_section = (
_build_ordinary_qa_markdown_format_section()
ordinary_qa_format_section = (
_build_ordinary_qa_markdown_format_section()
if _ordinary_qa_markdown_format_enabled()
and not is_bootstrap
and not writing_mode
@@ -136,13 +136,18 @@ def _build_runtime_middlewares(
# wrap_tool_call chain: its handler is the raw tool, so it observes the
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
# an error ToolMessage) and the raw return value.
# Omit the recorder entirely while paused: no record allocation, skill-merge
# lookup, or fire-and-forget DB task is created by any tool invocation.
# Keep tool execution/error handling and LLM metrics independent of this flag.
if app_config.runtime_performance.tool_metrics_enabled:
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
middlewares.append(ToolMetricsMiddleware())
# Full auditing can stay paused under high concurrency while the low-volume
# skill-only recorder remains active. In skill-only mode non-skill calls are
# passed through without a merge query or fire-and-forget DB write.
performance = app_config.runtime_performance
if performance.tool_metrics_enabled or performance.skill_metrics_enabled:
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
middlewares.append(
ToolMetricsMiddleware(
skill_only=not performance.tool_metrics_enabled,
)
)
return middlewares
@@ -179,20 +184,20 @@ def build_subagent_runtime_middlewares(
model_name = app_config.models[0].name
model_config = app_config.get_model_config(model_name) if model_name else None
if model_config is not None and model_config.supports_vision:
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
middlewares.append(ViewImageMiddleware())
# Must remain the innermost model-call wrapper: subagent-specific context,
# vision payloads, system prompts and tool schemas all need to be included
# in the final input-size decision.
from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware
middlewares.append(
ContextWindowGuardMiddleware(
max_input_tokens=model_config.max_input_tokens if model_config is not None else None,
)
)
return middlewares
if model_config is not None and model_config.supports_vision:
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
middlewares.append(ViewImageMiddleware())
# Must remain the innermost model-call wrapper: subagent-specific context,
# vision payloads, system prompts and tool schemas all need to be included
# in the final input-size decision.
from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware
middlewares.append(
ContextWindowGuardMiddleware(
max_input_tokens=model_config.max_input_tokens if model_config is not None else None,
)
)
return middlewares
@@ -1,6 +1,7 @@
"""Middleware that records per-tool-call metrics for the admin audit page.
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
While the administrator-controlled call-metrics switch is enabled, every tool
/ skill invocation ends with one row in ``tool_call_metrics``:
- Successes record the wall-clock duration.
- Failures record a coarse ``error_category`` plus a one-line message.
@@ -40,6 +41,7 @@ from langgraph.errors import GraphBubbleUp
from langgraph.prebuilt.tool_node import ToolCallRequest
from langgraph.types import Command
from deerflow.config.system_settings import get_call_metrics_settings
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
@@ -188,11 +190,18 @@ def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | Non
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
"""Wrap each tool call to time it and persist a metric row."""
"""Time tool calls, optionally persisting only attributable skill calls."""
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
def __init__(
self,
*,
store: ToolMetricsStore | None = None,
skill_only: bool = False,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self._store_override = store
self._skill_only = skill_only
# -- store / context ----------------------------------------------------
@@ -205,6 +214,12 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
return None
return make_tool_metrics_store(sf)
@staticmethod
def _metrics_enabled() -> bool:
"""Read the administrator switch at call time so no restart is needed."""
return bool(get_call_metrics_settings().enabled)
@staticmethod
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
runtime = getattr(request, "runtime", None)
@@ -280,7 +295,11 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ToolCallRequest], ToolMessage | Command],
) -> ToolMessage | Command:
record = self._build_base_record(request)
if is_internal_tool_name(record.tool_name):
if (
not self._metrics_enabled()
or is_internal_tool_name(record.tool_name)
or (self._skill_only and not record.skill_name)
):
return handler(request)
started = time.monotonic()
try:
@@ -303,7 +322,11 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
) -> ToolMessage | Command:
record = self._build_base_record(request)
if is_internal_tool_name(record.tool_name):
if (
not self._metrics_enabled()
or is_internal_tool_name(record.tool_name)
or (self._skill_only and not record.skill_name)
):
return await handler(request)
started = time.monotonic()
try:
@@ -482,13 +482,14 @@ def _load_and_cache_app_config(config_path: str | None = None) -> AppConfig:
)
performance = _app_config.runtime_performance
logger.info(
"RUNTIME_PERFORMANCE skill_view_reuse_enabled=%s skill_view_cache_enabled=%s model_timing_nonblocking_enabled=%s asyncio_thread_pool_enabled=%s asyncio_thread_pool_size=%d tool_metrics_enabled=%s",
"RUNTIME_PERFORMANCE skill_view_reuse_enabled=%s skill_view_cache_enabled=%s model_timing_nonblocking_enabled=%s asyncio_thread_pool_enabled=%s asyncio_thread_pool_size=%d tool_metrics_enabled=%s skill_metrics_enabled=%s",
performance.skill_view_reuse_enabled,
performance.skill_view_cache_enabled,
performance.model_timing_nonblocking_enabled,
performance.asyncio_thread_pool_enabled,
performance.asyncio_thread_pool_size,
performance.tool_metrics_enabled,
performance.skill_metrics_enabled,
)
return _app_config
@@ -2,9 +2,11 @@
Skill-view reuse and cache are on when those keys are absent. Set them to
false in config.yaml to restore a full refresh on every tool call. Model
timing and the Gateway thread pool stay off unless configured. If application
configuration cannot be loaded, every switch stays off. Tool-call metric writes
are paused by default; explicitly enable them to resume audit statistics.
timing and the Gateway thread pool stay off unless configured. Tool and skill
metrics are on by default so the management leaderboard has a complete source
of truth. Deployments that need to shed this best-effort audit write load can
explicitly pause either metric type in ``config.yaml``. If configuration
cannot be loaded, the fallback keeps every switch off.
"""
from pydantic import BaseModel, ConfigDict, Field
@@ -15,7 +17,14 @@ class RuntimePerformanceConfig(BaseModel):
skill_view_reuse_enabled: bool = Field(default=True, description="Reuse one verified skill view within each sandbox tool invocation. Set false to refresh on every call.")
skill_view_cache_enabled: bool = Field(default=True, description="Reuse unchanged skill views after validating source, target and availability versions. Set false to rebuild every call.")
tool_metrics_enabled: bool = Field(default=False, description="Persist tool/skill audit statistics to tool_call_metrics. Disabled by default to avoid competing with interactive database traffic; historical reads remain available.")
tool_metrics_enabled: bool = Field(
default=True,
description="Persist tool/skill audit statistics to tool_call_metrics. Enabled by default so management leaderboard tool rankings receive data; set false only to pause ordinary tool auditing.",
)
skill_metrics_enabled: bool = Field(
default=True,
description="Persist calls that can be attributed to a concrete skill, keeping skill statistics available even if ordinary tool auditing is paused.",
)
model_timing_nonblocking_enabled: bool = Field(default=False, description="Dispatch model timing logs through a dedicated bounded queue, outside the shared tool executor")
asyncio_thread_pool_enabled: bool = Field(default=False, description="Override the Gateway event loop's default executor at startup; disabled preserves Python defaults")
asyncio_thread_pool_size: int = Field(default=100, ge=1, le=1024, strict=True, description="Maximum Gateway executor threads when enabled, unless overridden by the documented environment variables")
@@ -24,7 +33,12 @@ class RuntimePerformanceConfig(BaseModel):
# Field defaults turn the skill-view fast path on. This fallback is only for
# callers that cannot read application configuration, and it keeps every
# switch off.
_DEFAULT = RuntimePerformanceConfig(skill_view_reuse_enabled=False, skill_view_cache_enabled=False)
_DEFAULT = RuntimePerformanceConfig(
skill_view_reuse_enabled=False,
skill_view_cache_enabled=False,
tool_metrics_enabled=False,
skill_metrics_enabled=False,
)
def get_runtime_performance_config() -> RuntimePerformanceConfig:
@@ -21,7 +21,7 @@ from pathlib import Path
from typing import Literal
from uuid import uuid4
from pydantic import BaseModel, ConfigDict, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride
from deerflow.config.runtime_paths import runtime_home
@@ -474,6 +474,20 @@ class SkillCompressionSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class SkillVersioningSettings(BaseModel):
"""Deployment-wide gate for custom-skill package version history.
The default is intentionally disabled so existing deployments retain the
legacy in-place editing behaviour until an administrator opts in.
"""
enabled: bool = Field(
default=False,
description="Whether custom-skill package versioning and switching are enabled",
)
model_config = ConfigDict(extra="ignore")
class AgentModuleSettings(BaseModel):
"""Admin-configured「按类型聚合」module → selected agent-tag ids.
@@ -668,6 +682,24 @@ class WriteFileCompatibilitySettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class CallMetricsSettings(BaseModel):
"""Administrator-controlled recording for tool and skill call statistics.
Both rankings use the same ``tool_call_metrics`` audit stream. Keeping a
single switch prevents a partially enabled configuration where the
management page can show one ranking but not the other.
"""
enabled: bool = Field(
default=True,
description=(
"Whether new tool and skill calls are persisted for management "
"statistics. Existing records are retained when disabled."
),
)
model_config = ConfigDict(extra="ignore")
class RetryKeepPartialSettings(BaseModel):
"""Keep already-streamed partial answers visible across silent LLM retries.
@@ -717,6 +749,15 @@ class RoundtableRoomLimitSettings(BaseModel):
"but never extend this deadline."
),
)
lease_minutes: int = Field(
default=5,
ge=1,
le=60,
description=(
"How long a foreground step-2 lease stays occupied without a renewal. "
"The browser heartbeat renews it; this does not extend max_hold_minutes."
),
)
max_cycles: int = Field(
default=8,
ge=2,
@@ -769,7 +810,40 @@ class HtmlMapSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
_GENERATED_PAGE_LOGO_FILE_RE = re.compile(r"^logo-[a-f0-9]{32}\.(?:png|jpg|webp)$")
class GeneratedPageBrandingSettings(BaseModel):
"""Optional generated-page logo settings retained for offline bundles."""
chrome_enabled: bool = True
header_layout: Literal["left", "center"] = "center"
system_name: str | None = Field(default=None, max_length=80)
footer_text: str | None = Field(default=None, max_length=160)
logo_filename: str | None = None
logo_mode: Literal["custom", "default", "none"] = "custom"
logo_media_type: Literal["image/png", "image/jpeg", "image/webp"] | None = None
logo_version: str | None = Field(default=None, max_length=64)
model_config = ConfigDict(extra="ignore")
@field_validator("logo_filename")
@classmethod
def _validate_logo_filename(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = str(value).strip().lower()
if not _GENERATED_PAGE_LOGO_FILE_RE.fullmatch(normalized):
raise ValueError("Invalid generated-page logo filename")
return normalized
_APP_IDENTITY_LOGO_FILE_RE = re.compile(r"^logo-[a-f0-9]{32}\.(?:png|jpg|webp)$")
_APP_IDENTITY_LIGHT_RED_HEADER_FILE_RE = re.compile(
r"^light-red-header-[a-f0-9]{32}\.(?:png|jpg|webp|svg)$"
)
_APP_IDENTITY_LIGHT_RED_ACTIVE_FILE_RE = re.compile(
r"^light-red-active-[a-f0-9]{32}\.(?:png|jpg|webp|svg)$"
)
class AppIdentitySettings(BaseModel):
@@ -780,9 +854,21 @@ class AppIdentitySettings(BaseModel):
"""
system_name: str = Field(default="", max_length=80)
header_title_font_mode: Literal["adaptive", "fixed"] = "adaptive"
header_title_font_size: int = Field(default=30, ge=16, le=40)
logo_filename: str | None = None
logo_media_type: Literal["image/png", "image/jpeg", "image/webp"] | None = None
logo_version: str | None = None
light_red_header_background_filename: str | None = None
light_red_header_background_media_type: Literal[
"image/png", "image/jpeg", "image/webp", "image/svg+xml"
] | None = None
light_red_header_background_version: str | None = None
light_red_nav_active_filename: str | None = None
light_red_nav_active_media_type: Literal[
"image/png", "image/jpeg", "image/webp", "image/svg+xml"
] | None = None
light_red_nav_active_version: str | None = None
model_config = ConfigDict(extra="ignore")
@field_validator("system_name")
@@ -801,6 +887,26 @@ class AppIdentitySettings(BaseModel):
raise ValueError("Invalid application logo filename")
return normalized
@field_validator("light_red_header_background_filename")
@classmethod
def _validate_light_red_header_filename(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = str(value).strip().lower()
if not _APP_IDENTITY_LIGHT_RED_HEADER_FILE_RE.fullmatch(normalized):
raise ValueError("Invalid light-red header background filename")
return normalized
@field_validator("light_red_nav_active_filename")
@classmethod
def _validate_light_red_active_filename(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = str(value).strip().lower()
if not _APP_IDENTITY_LIGHT_RED_ACTIVE_FILE_RE.fullmatch(normalized):
raise ValueError("Invalid light-red active navigation filename")
return normalized
def app_identity_logo_path(
settings: AppIdentitySettings | None = None,
@@ -814,6 +920,42 @@ def app_identity_logo_path(
return path if path.is_file() else None
def app_identity_light_red_header_background_path(
settings: AppIdentitySettings | None = None,
) -> Path | None:
"""Resolve the uploaded light-red header background inside runtime storage."""
current = settings or load_system_settings().app_identity
if not current.light_red_header_background_filename:
return None
path = runtime_home() / "app-identity" / current.light_red_header_background_filename
return path if path.is_file() else None
def app_identity_light_red_nav_active_path(
settings: AppIdentitySettings | None = None,
) -> Path | None:
"""Resolve the uploaded light-red active-navigation image inside runtime storage."""
current = settings or load_system_settings().app_identity
if not current.light_red_nav_active_filename:
return None
path = runtime_home() / "app-identity" / current.light_red_nav_active_filename
return path if path.is_file() else None
def generated_page_logo_path(
settings: GeneratedPageBrandingSettings | None = None,
) -> Path | None:
"""Resolve the custom generated-page logo without accepting arbitrary paths."""
current = settings or load_system_settings().generated_page_branding
if current.logo_mode != "custom" or not current.logo_filename:
return None
path = runtime_home() / "generated-page-branding" / current.logo_filename
return path if path.is_file() else None
class HtmlRuntimeSettings(BaseModel):
"""Addresses written into newly generated HTML pages.
@@ -844,6 +986,108 @@ class StrategyProjectSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class GeneralQaPageDescription(BaseModel):
"""Administrator-provided copy for the ordinary Q&A landing page."""
welcome_title: str = Field(default="", max_length=80)
welcome_description: str = Field(default="", max_length=500)
input_placeholder: str = Field(default="", max_length=120)
model_config = ConfigDict(extra="ignore")
@field_validator("welcome_title", "welcome_description", "input_placeholder")
@classmethod
def _normalize_copy(cls, value: str | None) -> str:
return str(value or "").strip()
class GeneralQaPageDescriptions(BaseModel):
"""Separate ordinary-Q&A copy for the full and compact layouts."""
full: GeneralQaPageDescription = Field(default_factory=GeneralQaPageDescription)
compact: GeneralQaPageDescription = Field(default_factory=GeneralQaPageDescription)
model_config = ConfigDict(extra="ignore")
@model_validator(mode="before")
@classmethod
def _migrate_legacy_single_description(cls, value: object) -> object:
if isinstance(value, GeneralQaPageDescription):
return {"full": value, "compact": value}
if not isinstance(value, dict):
return value
if "full" in value or "compact" in value:
return value
legacy = {
key: value.get(key, "")
for key in ("welcome_title", "welcome_description", "input_placeholder")
}
return {"full": legacy, "compact": legacy}
class PageHeaderDescription(BaseModel):
"""Configurable title and explanatory copy for a product landing page."""
title: str = Field(default="", max_length=80)
description: str = Field(default="", max_length=500)
model_config = ConfigDict(extra="ignore")
@field_validator("title", "description")
@classmethod
def _normalize_copy(cls, value: str | None) -> str:
return str(value or "").strip()
class AIWritingGuideCard(BaseModel):
"""Editable text for one of the fixed AI-writing welcome cards."""
title: str = Field(default="", max_length=80)
description: str = Field(default="", max_length=240)
model_config = ConfigDict(extra="ignore")
@field_validator("title", "description")
@classmethod
def _normalize_copy(cls, value: str | None) -> str:
return str(value or "").strip()
def _empty_ai_writing_guide_cards() -> list[AIWritingGuideCard]:
return [AIWritingGuideCard() for _ in range(3)]
class AIWritingPageDescription(PageHeaderDescription):
"""Configurable copy for the classic AI-writing welcome page."""
guide_cards: list[AIWritingGuideCard] = Field(
default_factory=_empty_ai_writing_guide_cards,
max_length=3,
)
@model_validator(mode="before")
@classmethod
def _keep_three_guide_cards(cls, value: object) -> object:
if not isinstance(value, dict) or not isinstance(value.get("guide_cards"), list):
return value
cards = value["guide_cards"][:3]
return {**value, "guide_cards": [*cards, *({} for _ in range(3 - len(cards)))]}
class PageDescriptionsSettings(BaseModel):
"""Page-scoped, administrator-managed display copy."""
general_qa: GeneralQaPageDescriptions = Field(default_factory=GeneralQaPageDescriptions)
knowledge_base: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
ai_writing: AIWritingPageDescription = Field(default_factory=AIWritingPageDescription)
strategy_projects: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
deep_research: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
gongwen: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
light_app_management: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
light_app_list: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
agents: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
business_chains: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
skills: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
knowledge_management: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
model_config = ConfigDict(extra="ignore")
class ModelPromptSuffix(BaseModel):
"""Administrator-owned instructions appended to one configured model's system prompt."""
@@ -862,9 +1106,14 @@ class SystemSettings(BaseModel):
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
publish_squares: PublishSquaresSettings = Field(default_factory=PublishSquaresSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
generated_page_branding: GeneratedPageBrandingSettings = Field(
default_factory=GeneratedPageBrandingSettings
)
page_descriptions: PageDescriptionsSettings = Field(default_factory=PageDescriptionsSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
skill_versioning: SkillVersioningSettings = Field(default_factory=SkillVersioningSettings)
plugin_management: PluginManagementSettings = Field(default_factory=PluginManagementSettings)
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
wx_common_agents: WxCommonAgentsSettings = Field(default_factory=WxCommonAgentsSettings)
@@ -875,6 +1124,7 @@ class SystemSettings(BaseModel):
write_file_compatibility: WriteFileCompatibilitySettings = Field(
default_factory=WriteFileCompatibilitySettings
)
call_metrics: CallMetricsSettings = Field(default_factory=CallMetricsSettings)
retry_keep_partial: RetryKeepPartialSettings = Field(
default_factory=RetryKeepPartialSettings
)
@@ -1007,6 +1257,19 @@ def get_write_file_compatibility_settings() -> WriteFileCompatibilitySettings:
return WriteFileCompatibilitySettings()
def get_call_metrics_settings() -> CallMetricsSettings:
"""Return the live call-metrics switch, defaulting to recording enabled."""
try:
return load_system_settings().call_metrics
except Exception:
logger.debug(
"Failed to read call metrics setting; keeping metrics enabled",
exc_info=True,
)
return CallMetricsSettings()
def get_roundtable_room_limit_settings() -> RoundtableRoomLimitSettings:
"""Return the admin-controlled roundtable room cap, defaulting to 3 enabled rooms."""
try:
@@ -0,0 +1,321 @@
"""Build portable ZIP bundles for generated HTML artifacts.
Generated pages intentionally reference the Gateway's vendored browser runtime
instead of public CDNs. A plain ``.html`` download therefore stops working
after it is moved away from the DeerFlow deployment. This module rewrites
those trusted Gateway URLs to relative paths and copies the corresponding
packaged assets into a ZIP. It never downloads arbitrary third-party URLs.
"""
from __future__ import annotations
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass
from importlib.resources import files
from pathlib import Path, PurePosixPath
from deerflow.html_runtime.public_images import resolve_html_image_path, verify_html_image_token
_STATIC_ASSETS = {
"alpine.min.js",
"bootstrap.bundle.min.js",
"bootstrap.min.css",
"brand-logo-dark.png",
"echarts.min.js",
"g6.min.js",
"leaflet.css",
"leaflet.js",
"topbar-dark.svg",
"topbar-light.svg",
}
_GEO_MAP_ASSETS = {
"china-provinces.geojson",
"japan-prefectures.geojson",
"taiwan-counties.geojson",
"us-states.geojson",
}
_COGNITIVE_ASSET_SUFFIXES = {
".css",
".js",
".json",
".jpeg",
".jpg",
".mjs",
".png",
".svg",
".woff2",
}
_LICENSE_BY_ASSET = {
"alpine.min.js": "LICENSE.alpine.txt",
"bootstrap.bundle.min.js": "LICENSE.bootstrap.txt",
"bootstrap.min.css": "LICENSE.bootstrap.txt",
"echarts.min.js": "LICENSE.echarts.txt",
"g6.min.js": "LICENSE.g6.txt",
"leaflet.css": "LICENSE.leaflet.txt",
"leaflet.js": "LICENSE.leaflet.txt",
}
_RUNTIME_ARCHIVE_ROOT = PurePosixPath("assets/html-runtime")
_HTML_IMAGES_ARCHIVE_ROOT = PurePosixPath("assets/html-images")
_STANDARD_RUNTIME_BASE_RE = (
r"(?:https?://[^/\"'\s<>()]+)?"
r"(?:/[A-Za-z0-9._~!$&()*+,;=:@%+-]+)*/api/public/html-runtime"
)
_HTML_IMAGE_URL_RE = re.compile(
r"(?P<url>(?:https?://[^/\"'\s<>()]+)?"
r"(?:/[A-Za-z0-9._~!$&()*+,;=:@%+-]+)*/api/public/html-images/"
r"(?P<token>[^\"'\s<>()?#]+)(?:\?[^\"'\s<>()#]*)?)",
re.IGNORECASE,
)
_EXTERNAL_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE)
@dataclass(frozen=True)
class OfflineBundleResult:
"""Summary of one completed offline bundle."""
bundled_assets: tuple[str, ...]
external_urls: tuple[str, ...]
def _runtime_base_pattern(runtime_base_url: str) -> str:
bases = [_STANDARD_RUNTIME_BASE_RE]
configured = str(runtime_base_url or "").strip().rstrip("/")
if configured:
bases.insert(0, re.escape(configured))
return "(?:" + "|".join(bases) + ")"
def _safe_runtime_source(runtime_dir: Path, relative_path: str) -> Path | None:
"""Resolve only assets that the public HTML-runtime router exposes."""
normalized = relative_path.strip().lstrip("/").replace("\\", "/")
pure_path = PurePosixPath(normalized)
if not normalized or pure_path.is_absolute() or ".." in pure_path.parts:
return None
allowed = False
if len(pure_path.parts) == 1 and normalized in _STATIC_ASSETS:
allowed = True
elif len(pure_path.parts) == 2 and pure_path.parts[0] == "maps" and pure_path.name in _GEO_MAP_ASSETS:
allowed = True
elif pure_path.parts and pure_path.parts[0] == "cognitive-situation-map":
allowed = pure_path.suffix.lower() in _COGNITIVE_ASSET_SUFFIXES
if not allowed:
return None
root = runtime_dir.resolve()
source = (root / Path(*pure_path.parts)).resolve()
try:
source.relative_to(root)
except ValueError:
return None
return source if source.is_file() else None
def _add_file(
archive_files: dict[str, Path],
archive_path: PurePosixPath,
source: Path,
) -> None:
archive_files[archive_path.as_posix()] = source
def _add_cognitive_runtime(archive_files: dict[str, Path], runtime_dir: Path) -> None:
source_root = runtime_dir / "cognitive-situation-map"
if not source_root.is_dir():
return
for source in source_root.rglob("*"):
if not source.is_file():
continue
is_license = source.name.lower().startswith("license")
if source.suffix.lower() not in _COGNITIVE_ASSET_SUFFIXES and not is_license:
continue
relative = source.relative_to(runtime_dir)
_add_file(archive_files, _RUNTIME_ARCHIVE_ROOT / PurePosixPath(relative.as_posix()), source)
def _generated_logo_destination(generated_logo_path: Path | None) -> PurePosixPath:
suffix = generated_logo_path.suffix.lower() if generated_logo_path is not None else ".png"
if suffix not in {".jpg", ".jpeg", ".png", ".webp"}:
suffix = ".png"
return _RUNTIME_ARCHIVE_ROOT / f"generated-page-logo{suffix}"
def _runtime_script(
runtime_dir: Path,
*,
map_tile_url: str,
include_geo_maps: bool,
) -> str:
config: dict[str, object] = {"geoMaps": {}, "mapTileUrl": map_tile_url}
if include_geo_maps:
config["geoMaps"] = {
Path(filename).stem: (_RUNTIME_ARCHIVE_ROOT / "maps" / filename).as_posix()
for filename in sorted(_GEO_MAP_ASSETS)
}
prefix = "window.DEERFLOW_HTML_RUNTIME_CONFIG=" + json.dumps(
config,
ensure_ascii=False,
separators=(",", ":"),
) + ";\n"
return prefix + (runtime_dir / "deerflow-runtime.js").read_text(encoding="utf-8")
def _readme(original_name: str, external_urls: list[str]) -> str:
external_note = (
"未打包的外部地址已记录在 manifest.json 的 external_urls 中;例如在线地图瓦片、"
"用户主动引用的第三方图片或接口仍需要网络。"
if external_urls
else "本页没有检测到仍需联网的 http(s) 地址。"
)
return (
"DeerFlow 网页离线包\n"
"====================\n\n"
f"原始文件:{original_name}\n"
"入口文件:index.html\n"
"依赖目录:assets/\n\n"
"部署方式:请保持目录结构不变,将解压后的全部文件一起放到任意静态网站目录。\n"
"若页面含 GeoJSON 地图等 fetch 请求,不建议直接双击 file:// 打开;可使用 Nginx、"
"Apache、IIS 或其他静态 HTTP 服务访问 index.html。\n\n"
f"{external_note}\n"
)
def build_html_offline_bundle(
html_path: Path,
destination: Path,
*,
runtime_base_url: str,
map_tile_url: str,
generated_logo_path: Path | None = None,
runtime_dir: Path | None = None,
) -> OfflineBundleResult:
"""Write a portable HTML ZIP to *destination* and return its manifest summary."""
html_path = Path(html_path)
destination = Path(destination)
runtime_dir = runtime_dir or Path(str(files("deerflow.html_runtime.assets")))
runtime_dir = Path(runtime_dir)
html = html_path.read_text(encoding="utf-8", errors="replace")
archive_files: dict[str, Path] = {}
generated_files: dict[str, str] = {}
bundled_assets: set[str] = set()
licenses: set[str] = set()
include_geo_maps = bool(re.search(r"data-df-geo\s*=", html, re.IGNORECASE))
uses_leaflet_map = bool(re.search(r"data-df-map\s*=", html, re.IGNORECASE))
base_pattern = _runtime_base_pattern(runtime_base_url)
cognitive_base_re = re.compile(
rf"(?P<url>{base_pattern}/cognitive-situation-map)(?=$|[\"'\s<>()/])",
re.IGNORECASE,
)
def replace_cognitive_base(match: re.Match[str]) -> str:
_add_cognitive_runtime(archive_files, runtime_dir)
bundled_assets.add((_RUNTIME_ARCHIVE_ROOT / "cognitive-situation-map").as_posix() + "/")
return (_RUNTIME_ARCHIVE_ROOT / "cognitive-situation-map").as_posix()
html = cognitive_base_re.sub(replace_cognitive_base, html)
runtime_url_re = re.compile(
rf"(?P<url>{base_pattern}/(?P<asset>[A-Za-z0-9._/-]+)"
r"(?:\?[^\"'\s<>()#]*)?)",
re.IGNORECASE,
)
def replace_runtime_url(match: re.Match[str]) -> str:
asset = match.group("asset").strip("/")
if asset == "generated-page-logo":
source = generated_logo_path if generated_logo_path and generated_logo_path.is_file() else runtime_dir / "brand-logo-dark.png"
if not source.is_file():
return match.group("url")
archive_path = _generated_logo_destination(source)
_add_file(archive_files, archive_path, source)
bundled_assets.add(archive_path.as_posix())
return archive_path.as_posix()
if asset == "deerflow-runtime.js":
source = runtime_dir / asset
if not source.is_file():
return match.group("url")
archive_path = _RUNTIME_ARCHIVE_ROOT / asset
generated_files[archive_path.as_posix()] = _runtime_script(
runtime_dir,
map_tile_url=map_tile_url,
include_geo_maps=include_geo_maps,
)
bundled_assets.add(archive_path.as_posix())
if include_geo_maps:
for filename in sorted(_GEO_MAP_ASSETS):
map_source = _safe_runtime_source(runtime_dir, f"maps/{filename}")
if map_source is not None:
map_archive_path = _RUNTIME_ARCHIVE_ROOT / "maps" / filename
_add_file(archive_files, map_archive_path, map_source)
bundled_assets.add(map_archive_path.as_posix())
return archive_path.as_posix()
source = _safe_runtime_source(runtime_dir, asset)
if source is None:
return match.group("url")
archive_path = _RUNTIME_ARCHIVE_ROOT / PurePosixPath(asset)
_add_file(archive_files, archive_path, source)
bundled_assets.add(archive_path.as_posix())
license_name = _LICENSE_BY_ASSET.get(asset)
if license_name and (runtime_dir / license_name).is_file():
licenses.add(license_name)
return archive_path.as_posix()
html = runtime_url_re.sub(replace_runtime_url, html)
def replace_html_image(match: re.Match[str]) -> str:
token = match.group("token")
try:
_thread_id, _kind, filename = verify_html_image_token(token)
source = resolve_html_image_path(_thread_id, _kind, filename)
except (FileNotFoundError, ValueError):
return match.group("url")
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()[:12]
safe_name = Path(filename).name
archive_path = _HTML_IMAGES_ARCHIVE_ROOT / f"{digest}-{safe_name}"
_add_file(archive_files, archive_path, source)
bundled_assets.add(archive_path.as_posix())
return archive_path.as_posix()
html = _HTML_IMAGE_URL_RE.sub(replace_html_image, html)
external_url_set = {
match.group(0).rstrip("),.;")
for match in _EXTERNAL_URL_RE.finditer(html)
}
if uses_leaflet_map and str(map_tile_url).lower().startswith(("http://", "https://")):
external_url_set.add(str(map_tile_url))
external_urls = sorted(external_url_set)
manifest = {
"format": "deerflow-html-offline-bundle-v1",
"entry": "index.html",
"source_name": html_path.name,
"bundled_assets": sorted(bundled_assets),
"external_urls": external_urls,
}
destination.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
archive.writestr("index.html", html.encode("utf-8"))
archive.writestr(
"manifest.json",
json.dumps(manifest, ensure_ascii=False, indent=2).encode("utf-8"),
)
archive.writestr("README.txt", _readme(html_path.name, external_urls).encode("utf-8"))
for archive_path, source in sorted(archive_files.items()):
archive.write(source, archive_path)
for archive_path, content in sorted(generated_files.items()):
archive.writestr(archive_path, content.encode("utf-8"))
for license_name in sorted(licenses):
archive.write(runtime_dir / license_name, f"THIRD_PARTY_LICENSES/{license_name}")
return OfflineBundleResult(
bundled_assets=tuple(sorted(bundled_assets)),
external_urls=tuple(external_urls),
)
@@ -36,7 +36,9 @@ from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
LEASE_SECONDS = 60
DEFAULT_LEASE_SECONDS = 5 * 60
MIN_LEASE_SECONDS = 60
MAX_LEASE_SECONDS = 60 * 60
JOB_LIVE_SECONDS = 15 * 60
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
# A background job keeps running while Step 3 generates its artifacts, but the
@@ -80,6 +82,15 @@ def _job_is_live(
return updated is not None and updated >= live_after
def normalize_lease_seconds(value: int | None) -> int:
"""Clamp a foreground room lease to the supported 1–60 minute window."""
try:
seconds = int(value) if value is not None else DEFAULT_LEASE_SECONDS
except (TypeError, ValueError):
seconds = DEFAULT_LEASE_SECONDS
return max(MIN_LEASE_SECONDS, min(MAX_LEASE_SECONDS, seconds))
def _hard_deadline_reached(
created_at: datetime | None,
now: datetime,
@@ -104,6 +115,7 @@ class RoundtableRoomRepository:
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically reserve/renew a room, or reject when the pool is full."""
@@ -119,6 +131,7 @@ class RoundtableRoomRepository:
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
lease_seconds=lease_seconds,
extra_keys=extra_keys,
)
@@ -133,6 +146,7 @@ class RoundtableRoomRepository:
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically admit a pre-resolved room key.
@@ -201,6 +215,7 @@ class RoundtableRoomRepository:
task_id=task_id,
holder_user_id=holder_user_id,
now=now,
lease_seconds=lease_seconds,
)
await session.commit()
return self._decision(
@@ -272,6 +287,7 @@ class RoundtableRoomRepository:
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew one live foreground lease without ever creating it.
@@ -298,6 +314,7 @@ class RoundtableRoomRepository:
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
lease_seconds=lease_seconds,
extra_keys=extra_keys,
)
@@ -309,6 +326,7 @@ class RoundtableRoomRepository:
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew a resolved persisted key without creating a missing lease."""
@@ -354,7 +372,7 @@ class RoundtableRoomRepository:
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
)
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
row.lease_until = now + timedelta(seconds=normalize_lease_seconds(lease_seconds))
row.updated_at = now
await session.commit()
return self._decision(
@@ -538,9 +556,10 @@ class RoundtableRoomRepository:
task_id: str | None,
holder_user_id: str | None,
now: datetime,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
) -> None:
row = await session.get(RoundtableRoomLeaseRow, room_key)
lease_until = now + timedelta(seconds=LEASE_SECONDS)
lease_until = now + timedelta(seconds=normalize_lease_seconds(lease_seconds))
if row is None:
session.add(
RoundtableRoomLeaseRow(
@@ -314,43 +314,41 @@ def _resolve_skills_path(path: str) -> str:
return _join_path_preserving_style(skills_host, relative)
def _custom_skill_name_from_path(path: str) -> str | None:
"""从虚拟技能路径里解析出 custom 技能名,用于"被使用"埋点。
只识别 ``{skills_container}/custom/{name}/...`` 形式 —— public 技能不参与
策展,给它们埋点只会往 ``.usage.json`` 写无用条目。读技能目录下任意文件
(SKILL.md 或 references/templates/scripts 支持文件)都算一次该技能的使用。
解析不出技能名时返回 None。
"""
skills_container = _get_skills_container_path()
if not (path == skills_container or path.startswith(f"{skills_container}/")):
return None
relative = path[len(skills_container) :].lstrip("/")
parts = [segment for segment in relative.split("/") if segment]
if len(parts) >= 2 and parts[0] == "custom":
return parts[1]
return None
def _record_skill_read(virtual_path: str) -> None:
"""技能文件被 ``read_file`` 读取时记一次 use —— 技能"被使用"的真实信号。
技能正文不进 system prompt(prompt 里只有 name/description/location),Agent
按 Progressive Loading 用 ``read_file`` 加载 SKILL.md 才算真正用到该技能。
curator 的年龄流转据此判断闲置,没有这个埋点会把每天都在用的技能误判为陈旧。
best-effort:埋点失败绝不影响 ``read_file`` 本身。
"""
def _record_skill_paths(
virtual_paths: list[str],
runtime: ToolRuntime[ContextT, ThreadState] | None,
) -> None:
"""Record public/custom skills from real tool paths, independent of prose."""
try:
name = _custom_skill_name_from_path(virtual_path)
if name is None:
return
from deerflow.skills.usage import bump_use
from deerflow.sandbox.skill_view import skill_names_for_paths
from deerflow.skills.usage import bump_use, runtime_use_event_key
bump_use(name)
event_key = runtime_use_event_key(runtime)
for name in skill_names_for_paths(
virtual_paths,
container_path=_get_skills_container_path(),
):
bump_use(name, event_key=event_key)
except Exception:
pass
def _record_skill_read(
virtual_path: str,
runtime: ToolRuntime[ContextT, ThreadState] | None = None,
) -> None:
"""Count a successful skill file read as a logical skill invocation."""
_record_skill_paths([virtual_path], runtime)
def _record_skill_command(
command: str,
runtime: ToolRuntime[ContextT, ThreadState] | None = None,
) -> None:
"""Count skills whose scripts are executed directly by ``bash``."""
_record_skill_paths(_skill_paths_in_text(command), runtime)
def _is_acp_workspace_path(path: str) -> bool:
"""Check if a path is under the ACP workspace virtual path."""
return path == _ACP_WORKSPACE_VIRTUAL_PATH or path.startswith(f"{_ACP_WORKSPACE_VIRTUAL_PATH}/")
@@ -1445,6 +1443,7 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
command: The bash command to execute. Always use absolute paths for files and directories.
"""
try:
requested_command = command
_prepare_command_skill_capability(command)
_validate_enabled_skill_paths_in_command(command)
sandbox = ensure_sandbox_initialized(runtime)
@@ -1457,6 +1456,7 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
command = replace_virtual_paths_in_command(command, thread_data)
command = _apply_cwd_prefix(command, thread_data)
output = sandbox.execute_command(command)
_record_skill_command(requested_command, runtime)
try:
from deerflow.config.app_config import get_app_config
@@ -1473,7 +1473,9 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
max_chars = sandbox_cfg.bash_output_max_chars if sandbox_cfg else 20000
except Exception:
max_chars = 20000
return _truncate_bash_output(sandbox.execute_command(command), max_chars)
output = sandbox.execute_command(command)
_record_skill_command(requested_command, runtime)
return _truncate_bash_output(output, max_chars)
except SandboxError as e:
return f"Error: {e}"
except PermissionError as e:
@@ -1751,7 +1753,7 @@ def read_file_tool(
max_result_chars=max_chars,
hard_max_result_chars=hard_max,
)
_record_skill_read(requested_path)
_record_skill_read(requested_path, runtime)
return inspected["rendered"]
except OSError:
pass
@@ -1767,7 +1769,7 @@ def read_file_tool(
)
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
# 非技能路径会被 _record_skill_read 自行忽略。
_record_skill_read(requested_path)
_record_skill_read(requested_path, runtime)
if not content and not truncated:
return "(empty)"
if truncated:
@@ -7,6 +7,7 @@ import logging
import os
import tempfile
import threading
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
@@ -19,6 +20,14 @@ _USAGE_FILENAME = ".usage.json"
# 单进程跑,故进程内锁即可保证状态流转不丢。
_usage_lock = threading.Lock()
# One user run can first load SKILL.md and then execute several helper scripts.
# Those are implementation details of one logical skill invocation, so retain a
# small process-local LRU keyed by (run, skill) and increment the public counter
# only once. A run stays on one worker; persistence across restarts is neither
# needed nor desirable here.
_MAX_RECENT_USE_EVENTS = 8192
_recent_use_events: OrderedDict[tuple[str, str], None] = OrderedDict()
STATE_ACTIVE = "active"
STATE_STALE = "stale"
STATE_ARCHIVED = "archived"
@@ -121,8 +130,110 @@ def bump_view(name: str) -> None:
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
def bump_use(name: str) -> None:
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
async def _increment_use_projection(name: str, use_count: int, used_at: datetime) -> None:
"""Keep the MySQL skill-list projection current without importing ``app``."""
try:
from sqlalchemy import case, update
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.skills.model import SkillRow
session_factory = get_session_factory()
if session_factory is None:
return
async with session_factory() as session:
await session.execute(
update(SkillRow)
.where(SkillRow.name == name)
.values(
# Fire-and-forget writes may finish out of order. Never let
# an older absolute sidecar count overwrite a newer value.
use_count=case((SkillRow.use_count < use_count, use_count), else_=SkillRow.use_count),
last_activity_at=case(
(SkillRow.last_activity_at.is_(None), used_at),
(SkillRow.last_activity_at < used_at, used_at),
else_=SkillRow.last_activity_at,
),
updated_at=case(
(SkillRow.updated_at < used_at, used_at),
else_=SkillRow.updated_at,
),
)
)
await session.commit()
except Exception:
logger.debug("Failed to update skill usage projection for %r", name, exc_info=True)
def _schedule_use_projection_increment(name: str, use_count: int, used_at: datetime) -> None:
"""Best-effort bridge from synchronous sandbox tools to the DB home loop."""
try:
from deerflow.persistence.engine import fire_and_forget_db_write
fire_and_forget_db_write(_increment_use_projection(name, use_count, used_at))
except Exception:
logger.debug("Failed to schedule skill usage projection for %r", name, exc_info=True)
def runtime_use_event_key(runtime: Any | None) -> str | None:
"""Return the stable user-run identifier carried by ``ToolRuntime``."""
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in ("run_id", "langgraph_run_id"):
value = context.get(key)
if value is not None and str(value).strip():
return str(value).strip()
config = getattr(runtime, "config", None)
if isinstance(config, dict):
for bucket_name in ("configurable", "metadata"):
bucket = config.get(bucket_name)
if not isinstance(bucket, dict):
continue
for key in ("run_id", "langgraph_run_id"):
value = bucket.get(key)
if value is not None and str(value).strip():
return str(value).strip()
return None
def bump_use(name: str, *, event_key: str | None = None) -> bool:
"""Count one logical use, optionally de-duplicated by user run.
Returns ``True`` only when the counter changed. This makes callers and tests
able to distinguish a new invocation from a second file/script access in
the same run.
"""
normalized_name = str(name or "").strip()
if not normalized_name:
return False
normalized_event = str(event_key or "").strip() or None
dedupe_key = (normalized_event, normalized_name) if normalized_event else None
used_at = datetime.now(UTC)
new_count = 0
try:
with _usage_lock:
if dedupe_key is not None and dedupe_key in _recent_use_events:
_recent_use_events.move_to_end(dedupe_key)
return False
data = _read_all()
entry = {**_DEFAULT_ENTRY, **data.get(normalized_name, {})}
new_count = max(0, int(entry.get("use_count") or 0)) + 1
entry.update({"use_count": new_count, "last_used_at": used_at.isoformat()})
data[normalized_name] = entry
_write_all(data)
if dedupe_key is not None:
_recent_use_events[dedupe_key] = None
_recent_use_events.move_to_end(dedupe_key)
while len(_recent_use_events) > _MAX_RECENT_USE_EVENTS:
_recent_use_events.popitem(last=False)
except Exception:
logger.debug("Failed to bump usage entry for %r", normalized_name, exc_info=True)
return False
_schedule_use_projection_increment(normalized_name, new_count, used_at)
return True
def bump_patch(name: str) -> None:
@@ -0,0 +1,397 @@
"""Filesystem-backed version history for custom skills.
Only the active package lives below ``skills/custom``. Historical snapshots
are kept under the writable DeerFlow runtime directory, outside the canonical
skills tree and therefore outside every model-facing discovery/mount path.
The feature is deliberately gated by ``system_settings.skill_versioning``.
When the switch is off, callers keep the legacy write behaviour and this
module performs no snapshot writes.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import tempfile
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.skills.storage import SkillStorage, get_or_new_skill_storage
from deerflow.skills.types import SKILL_MD_FILE, SkillCategory
_MANIFEST_FILE = "manifest.json"
_SCHEMA_VERSION = 1
_LOCKS_GUARD = threading.Lock()
_THREAD_LOCKS: dict[str, threading.RLock] = {}
def skill_versioning_enabled() -> bool:
"""Return the deployment-wide custom-skill versioning switch."""
try:
return bool(load_system_settings().skill_versioning.enabled)
except Exception:
return False
def _thread_lock(name: str) -> threading.RLock:
with _LOCKS_GUARD:
return _THREAD_LOCKS.setdefault(name, threading.RLock())
def _versions_root() -> Path:
return get_paths().base_dir / "skill-versions"
def _skill_root(name: str) -> Path:
return _versions_root() / SkillStorage.validate_skill_name(name)
def _manifest_path(name: str) -> Path:
return _skill_root(name) / _MANIFEST_FILE
def _version_dir(name: str, version: int) -> Path:
return _skill_root(name) / f"v{version:06d}"
@contextmanager
def _cross_process_lock(name: str) -> Iterator[None]:
"""Serialize one skill's snapshot/switch operations across workers."""
root = _skill_root(name)
root.mkdir(parents=True, exist_ok=True)
lock_path = root / ".lock"
with _thread_lock(name):
with lock_path.open("a+b") as handle:
if os.name == "nt": # pragma: no cover - exercised on Windows hosts
import msvcrt
if lock_path.stat().st_size == 0:
handle.write(b"0")
handle.flush()
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
else: # pragma: no cover - exercised in Linux CI/deployments
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
if os.name == "nt": # pragma: no cover
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else: # pragma: no cover
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def _empty_manifest(name: str) -> dict[str, Any]:
return {
"schema_version": _SCHEMA_VERSION,
"skill_name": name,
"active_version": None,
"versions": [],
}
def _load_manifest(name: str) -> dict[str, Any]:
path = _manifest_path(name)
if not path.is_file():
return _empty_manifest(name)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ValueError(f"Skill version manifest for '{name}' is unreadable") from exc
if not isinstance(payload, dict) or payload.get("skill_name") != name:
raise ValueError(f"Skill version manifest for '{name}' is invalid")
versions = payload.get("versions")
if not isinstance(versions, list):
raise ValueError(f"Skill version manifest for '{name}' has invalid versions")
return payload
def _save_manifest(name: str, manifest: dict[str, Any]) -> None:
path = _manifest_path(name)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_name = tempfile.mkstemp(prefix=".manifest-", suffix=".tmp", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(manifest, handle, ensure_ascii=False, indent=2, sort_keys=True)
handle.write("\n")
os.replace(tmp_name, path)
except Exception:
try:
os.unlink(tmp_name)
except OSError:
pass
raise
def _resolve_custom_dir(storage: SkillStorage, name: str) -> Path:
for skill in storage.load_skills(enabled_only=False):
if skill.name == name and skill.category == SkillCategory.CUSTOM:
return skill.skill_dir.resolve()
raise FileNotFoundError(f"Custom skill '{name}' not found")
def _tree_hash(root: Path) -> str:
digest = hashlib.sha256()
for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()):
relative = path.relative_to(root).as_posix()
if any(part in {"__pycache__", ".git"} for part in path.relative_to(root).parts):
continue
if path.is_symlink():
digest.update(b"L\0")
digest.update(relative.encode("utf-8"))
digest.update(b"\0")
digest.update(os.readlink(path).encode("utf-8", errors="surrogateescape"))
continue
if not path.is_file():
continue
digest.update(b"F\0")
digest.update(relative.encode("utf-8"))
digest.update(b"\0")
with path.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _copy_package(source: Path, destination: Path) -> None:
shutil.copytree(
source,
destination,
symlinks=True,
ignore=shutil.ignore_patterns("__pycache__", ".git"),
)
def record_custom_skill_version(
name: str,
*,
storage: SkillStorage | None = None,
action: str,
author: str | None = None,
summary: str | None = None,
) -> dict[str, Any] | None:
"""Snapshot the current custom package when it differs from active.
Returns the active/new version metadata. No-op (and no filesystem writes)
while the deployment switch is disabled.
"""
if not skill_versioning_enabled():
return None
name = SkillStorage.validate_skill_name(name)
storage = storage or get_or_new_skill_storage()
source = _resolve_custom_dir(storage, name)
current_hash = _tree_hash(source)
with _cross_process_lock(name):
manifest = _load_manifest(name)
active = next(
(item for item in manifest["versions"] if int(item.get("version") or 0) == int(manifest.get("active_version") or 0)),
None,
)
if active is not None and active.get("sha256") == current_hash:
return {**active, "active": True}
next_version = max([int(item.get("version") or 0) for item in manifest["versions"]] or [0]) + 1
destination = _version_dir(name, next_version)
staging = destination.parent / f".{destination.name}.staging-{uuid4().hex}"
_copy_package(source, staging)
try:
os.replace(staging, destination)
except Exception:
shutil.rmtree(staging, ignore_errors=True)
raise
now = datetime.now(UTC).isoformat()
entry = {
"version": next_version,
"sha256": current_hash,
"created_at": now,
"created_by": author,
"action": str(action or "update"),
"summary": str(summary or "").strip() or None,
"based_on_version": manifest.get("active_version"),
}
manifest["versions"].append(entry)
manifest["active_version"] = next_version
manifest["updated_at"] = now
_save_manifest(name, manifest)
return {**entry, "active": True}
def list_custom_skill_versions(
name: str,
*,
storage: SkillStorage | None = None,
author: str | None = None,
) -> dict[str, Any]:
"""Return newest-first metadata, lazily bootstrapping the current package."""
if not skill_versioning_enabled():
return {"enabled": False, "skill_name": name, "active_version": None, "versions": []}
storage = storage or get_or_new_skill_storage()
record_custom_skill_version(
name,
storage=storage,
action="baseline",
author=author,
summary="启用技能迭代时保存的当前版本",
)
with _cross_process_lock(name):
manifest = _load_manifest(name)
active_version = manifest.get("active_version")
versions = [{**item, "active": int(item.get("version") or 0) == int(active_version or 0)} for item in reversed(manifest["versions"])]
return {
"enabled": True,
"skill_name": name,
"active_version": active_version,
"versions": versions,
}
def activate_custom_skill_version(
name: str,
version: int,
*,
storage: SkillStorage | None = None,
) -> dict[str, Any]:
"""Materialize one historical snapshot as the sole model-visible package."""
if not skill_versioning_enabled():
raise RuntimeError("技能迭代功能未开启")
name = SkillStorage.validate_skill_name(name)
storage = storage or get_or_new_skill_storage()
target = _resolve_custom_dir(storage, name)
with _cross_process_lock(name):
manifest = _load_manifest(name)
entry = next(
(item for item in manifest["versions"] if int(item.get("version") or 0) == int(version)),
None,
)
if entry is None:
raise FileNotFoundError(f"Skill '{name}' version {version} not found")
source = _version_dir(name, int(version))
if not source.is_dir() or _tree_hash(source) != entry.get("sha256"):
raise ValueError(f"Skill '{name}' version {version} snapshot is missing or corrupted")
storage.validate_skill_markdown_content(
name,
(source / SKILL_MD_FILE).read_text(encoding="utf-8"),
)
if int(manifest.get("active_version") or 0) == int(version) and _tree_hash(target) == entry.get("sha256"):
return {**entry, "active": True}
parent = target.parent
staging = parent / f".{name}.version-stage-{uuid4().hex}"
backup = parent / f".{name}.version-backup-{uuid4().hex}"
_copy_package(source, staging)
moved_old = False
try:
os.replace(target, backup)
moved_old = True
os.replace(staging, target)
manifest["active_version"] = int(version)
manifest["updated_at"] = datetime.now(UTC).isoformat()
_save_manifest(name, manifest)
except Exception:
shutil.rmtree(staging, ignore_errors=True)
if moved_old:
shutil.rmtree(target, ignore_errors=True)
os.replace(backup, target)
raise
finally:
if backup.exists() and target.exists():
shutil.rmtree(backup, ignore_errors=True)
return {**entry, "active": True}
def delete_custom_skill_versions(name: str) -> None:
"""Remove stored snapshots after the custom skill itself is deleted."""
name = SkillStorage.validate_skill_name(name)
root = _skill_root(name)
if root.exists():
shutil.rmtree(root)
def reset_custom_skill_versions(
name: str,
*,
storage: SkillStorage | None = None,
author: str | None = None,
summary: str | None = None,
) -> dict[str, Any] | None:
"""Discard the previous chain and make the current package the new v1.
This is the explicit "direct overwrite" semantic offered beside
"iterate as a new version" for same-name uploads.
"""
if not skill_versioning_enabled():
return None
name = SkillStorage.validate_skill_name(name)
root = _skill_root(name)
with _cross_process_lock(name):
for child in list(root.iterdir()):
if child.name == ".lock":
continue
if child.is_dir():
shutil.rmtree(child)
else:
child.unlink(missing_ok=True)
return record_custom_skill_version(
name,
storage=storage,
action="overwrite",
author=author,
summary=summary or "直接覆盖后重置版本链",
)
def reconcile_custom_skill_versions(*, storage: SkillStorage | None = None, author: str | None = None) -> int:
"""Capture each current custom skill after the master switch is enabled."""
if not skill_versioning_enabled():
return 0
storage = storage or get_or_new_skill_storage()
captured = 0
for skill in storage.load_skills(enabled_only=False):
if skill.category != SkillCategory.CUSTOM:
continue
before = _load_manifest(skill.name).get("active_version") if _manifest_path(skill.name).exists() else None
result = record_custom_skill_version(
skill.name,
storage=storage,
action="feature_enabled",
author=author,
summary="技能迭代开关开启后捕获当前内容",
)
if result is not None and result.get("version") != before:
captured += 1
return captured
__all__ = [
"activate_custom_skill_version",
"delete_custom_skill_versions",
"list_custom_skill_versions",
"record_custom_skill_version",
"reconcile_custom_skill_versions",
"reset_custom_skill_versions",
"skill_versioning_enabled",
]
@@ -273,7 +273,16 @@ def skill_view_tool(name: str, runtime: ToolRuntime) -> str:
Args:
name: The skill name (hyphen-case, e.g. 'my-skill').
"""
return skill_view_impl(name, _resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
result = skill_view_impl(name, _resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
head = result.lstrip()[:200].lower()
if not ((head.startswith("skill '") and "not found" in head) or head.startswith("error reading skill")):
try:
from deerflow.skills.usage import bump_use, runtime_use_event_key
bump_use(name, event_key=runtime_use_event_key(runtime))
except Exception:
pass
return result
def _resolve_skills_container_base_path() -> str:
@@ -161,6 +161,26 @@ async def _publish_managed_skill(name: str, *, removed: bool = False) -> None:
await _sync_managed_skill_view(name)
async def _record_managed_skill_version(
storage: SkillStorage,
name: str,
*,
action: str,
author: str,
summary: str,
) -> None:
from deerflow.skills.versioning import record_custom_skill_version
await _to_thread(
record_custom_skill_version,
name,
storage=storage,
action=action,
author=author,
summary=summary,
)
async def _bump_patch_best_effort(name: str) -> None:
"""记一次技能维护活动(patch / edit)。
@@ -363,6 +383,14 @@ async def _skill_manage_impl(
# async stack — contextvars don't propagate into ``to_thread``.
if skill_store is not None:
await _ensure_skill_db_registered(skill_store, name, user_id)
skill_storage.invalidate_skills_cache()
await _record_managed_skill_version(
skill_storage,
name,
action="agent_create",
author=user_id,
summary="大模型创建技能",
)
await _publish_managed_skill(name)
return f"Created custom skill '{name}'."
@@ -378,6 +406,13 @@ async def _skill_manage_impl(
scan = await _scan_or_raise(content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
skill_file = skill_storage.get_custom_skill_file(name)
prev_content = await _to_thread(skill_file.read_text, encoding="utf-8")
await _record_managed_skill_version(
skill_storage,
name,
action="baseline",
author=user_id,
summary="大模型编辑前的活动版本",
)
await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, content)
await _to_thread(
skill_storage.append_history,
@@ -385,6 +420,13 @@ async def _skill_manage_impl(
_history_record(action="edit", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
)
await _bump_patch_best_effort(name)
await _record_managed_skill_version(
skill_storage,
name,
action="agent_edit",
author=user_id,
summary="大模型编辑 SKILL.md",
)
await _publish_managed_skill(name)
return f"Updated custom skill '{name}'."
@@ -406,6 +448,13 @@ async def _skill_manage_impl(
new_content = prev_content.replace(find, replace, replacement_count)
await _to_thread(skill_storage.validate_skill_markdown_content, name, new_content)
scan = await _scan_or_raise(new_content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
await _record_managed_skill_version(
skill_storage,
name,
action="baseline",
author=user_id,
summary="大模型修补前的活动版本",
)
await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, new_content)
await _to_thread(
skill_storage.append_history,
@@ -413,6 +462,13 @@ async def _skill_manage_impl(
_history_record(action="patch", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=new_content, thread_id=thread_id, scanner=scan),
)
await _bump_patch_best_effort(name)
await _record_managed_skill_version(
skill_storage,
name,
action="agent_patch",
author=user_id,
summary="大模型修补 SKILL.md",
)
await _publish_managed_skill(name)
return f"Patched custom skill '{name}' ({replacement_count} replacement(s) applied, {occurrences} match(es) found)."
@@ -453,6 +509,12 @@ async def _skill_manage_impl(
await skill_store.delete(name, user_id)
except Exception:
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
try:
from deerflow.skills.versioning import delete_custom_skill_versions
await _to_thread(delete_custom_skill_versions, name)
except Exception:
logger.debug("Failed to delete version history for %r", name, exc_info=True)
agent_store = _default_agent_store()
if agent_store is not None:
try:
@@ -482,12 +544,26 @@ async def _skill_manage_impl(
prev_content = await _to_thread(target.read_text, encoding="utf-8") if exists else None
executable = "scripts/" in path or path.startswith("scripts/")
scan = await _scan_or_raise(content, executable=executable, location=f"{name}/{path}")
await _record_managed_skill_version(
skill_storage,
name,
action="baseline",
author=user_id,
summary="大模型写文件前的活动版本",
)
await _to_thread(skill_storage.write_custom_skill, name, path, content)
await _to_thread(
skill_storage.append_history,
name,
_history_record(action="write_file", file_path=path, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
)
await _record_managed_skill_version(
skill_storage,
name,
action="agent_write_file",
author=user_id,
summary=f"大模型写入文件 {path}",
)
await _sync_managed_skill_view(name)
return f"Wrote '{path}' for custom skill '{name}'."
@@ -502,12 +578,26 @@ async def _skill_manage_impl(
if not await _to_thread(target.exists):
raise FileNotFoundError(f"Supporting file '{path}' not found for skill '{name}'.")
prev_content = await _to_thread(target.read_text, encoding="utf-8")
await _record_managed_skill_version(
skill_storage,
name,
action="baseline",
author=user_id,
summary="大模型删文件前的活动版本",
)
await _to_thread(target.unlink)
await _to_thread(
skill_storage.append_history,
name,
_history_record(action="remove_file", file_path=path, prev_content=prev_content, new_content=None, thread_id=thread_id, scanner={"decision": "allow", "reason": "Deletion requested."}),
)
await _record_managed_skill_version(
skill_storage,
name,
action="agent_remove_file",
author=user_id,
summary=f"大模型删除文件 {path}",
)
await _sync_managed_skill_view(name)
return f"Removed '{path}' from custom skill '{name}'."
@@ -5,26 +5,28 @@ import pytest
from deerflow.config.app_config import AppConfig
_SKILL_VIEW_KEYS = ("skill_view_reuse_enabled", "skill_view_cache_enabled")
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled", "tool_metrics_enabled")
_DEFAULT_ON_KEYS = (*_SKILL_VIEW_KEYS, "tool_metrics_enabled", "skill_metrics_enabled")
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled")
def test_missing_runtime_performance_enables_skill_view_only():
def test_missing_runtime_performance_enables_skill_view_and_skill_metrics():
config = AppConfig(sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"})
assert config.runtime_performance.model_dump() == {
"skill_view_reuse_enabled": True,
"skill_view_cache_enabled": True,
"tool_metrics_enabled": False,
"tool_metrics_enabled": True,
"skill_metrics_enabled": True,
"model_timing_nonblocking_enabled": False,
"asyncio_thread_pool_enabled": False,
"asyncio_thread_pool_size": 100,
}
@pytest.mark.parametrize("key", [*_SKILL_VIEW_KEYS, *_OPT_IN_KEYS])
@pytest.mark.parametrize("key", [*_DEFAULT_ON_KEYS, *_OPT_IN_KEYS])
def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
# Skill-view switches default on, so independence is checked by turning
# one of them off. The opt-in switches default off and are turned on.
enabled = key not in _SKILL_VIEW_KEYS
# Skill-view and audit metrics default on, so independence is checked by
# turning one off. Opt-in switches default off and are turned on.
enabled = key not in _DEFAULT_ON_KEYS
config_path = tmp_path / "config.yaml"
config_path.write_text(
f"sandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\nruntime_performance:\n {key}: {'true' if enabled else 'false'}\n",
@@ -32,7 +34,7 @@ def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
)
flags = AppConfig.from_file(str(config_path)).runtime_performance.model_dump()
assert flags[key] is enabled
for other in _SKILL_VIEW_KEYS:
for other in _DEFAULT_ON_KEYS:
if other != key:
assert flags[other] is True
for other in _OPT_IN_KEYS:
@@ -0,0 +1,95 @@
"""Regression coverage for model-independent skill usage attribution."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from deerflow.sandbox import tools
from deerflow.skills import usage
from deerflow.tools.builtins import skill_tools
@pytest.fixture(autouse=True)
def _clear_recent_use_events():
usage._recent_use_events.clear()
yield
usage._recent_use_events.clear()
def test_bump_use_counts_a_skill_once_per_run_and_syncs_the_catalog(tmp_path: Path, monkeypatch):
usage_path = tmp_path / ".usage.json"
projected: list[tuple[str, int, object]] = []
monkeypatch.setattr(usage, "_usage_path", lambda: usage_path)
monkeypatch.setattr(
usage,
"_schedule_use_projection_increment",
lambda name, use_count, used_at: projected.append((name, use_count, used_at)),
)
assert usage.bump_use("page-generator", event_key="run-1") is True
assert usage.bump_use("page-generator", event_key="run-1") is False
assert usage.bump_use("page-generator", event_key="run-2") is True
entry = usage.get_entry("page-generator")
assert entry["use_count"] == 2
assert entry["last_used_at"]
assert [(name, count) for name, count, _used_at in projected] == [
("page-generator", 1),
("page-generator", 2),
]
def test_skill_file_and_script_paths_record_public_and_custom_skills(monkeypatch):
calls: list[tuple[str, str | None]] = []
monkeypatch.setattr(
usage,
"bump_use",
lambda name, *, event_key=None: calls.append((name, event_key)),
)
runtime = SimpleNamespace(context={"run_id": "run-weak-model"}, config={})
tools._record_skill_read("/mnt/skills/public/report-maker/SKILL.md", runtime)
tools._record_skill_command(
"python /mnt/skills/custom/data-cleaner/scripts/run.py --fast",
runtime,
)
assert calls == [
("report-maker", "run-weak-model"),
("data-cleaner", "run-weak-model"),
]
def test_skill_view_counts_as_use_without_model_narration(monkeypatch):
calls: list[tuple[str, str | None]] = []
runtime = SimpleNamespace(context={"run_id": "run-skill-view"}, config={})
monkeypatch.setattr(skill_tools, "skill_view_impl", lambda *_args, **_kwargs: "skill instructions")
monkeypatch.setattr(
usage,
"bump_use",
lambda name, *, event_key=None: calls.append((name, event_key)),
)
result = skill_tools.skill_view_tool.func(name="report-maker", runtime=runtime)
assert result == "skill instructions"
assert calls == [("report-maker", "run-skill-view")]
def test_failed_skill_view_does_not_count_as_use(monkeypatch):
calls: list[str] = []
runtime = SimpleNamespace(context={"run_id": "run-missing"}, config={})
monkeypatch.setattr(
skill_tools,
"skill_view_impl",
lambda *_args, **_kwargs: "Skill 'missing' not found.",
)
monkeypatch.setattr(usage, "bump_use", lambda name, **_kwargs: calls.append(name))
result = skill_tools.skill_view_tool.func(name="missing", runtime=runtime)
assert result == "Skill 'missing' not found."
assert calls == []
@@ -288,7 +288,7 @@ def _tool_sandbox(tmp_path: Path, view_root: Path, monkeypatch: pytest.MonkeyPat
monkeypatch.setattr(tools, "is_local_sandbox", lambda _runtime: True)
monkeypatch.setattr(tools, "is_host_bash_allowed", lambda: True)
monkeypatch.setattr(tools, "get_thread_data", lambda _runtime: {"workspace_path": str(tmp_path / "workspace"), "uploads_path": str(tmp_path / "uploads"), "outputs_path": str(tmp_path / "outputs")})
monkeypatch.setattr(tools, "_record_skill_read", lambda _path: None)
monkeypatch.setattr(tools, "_record_skill_read", lambda *_args: None)
monkeypatch.setattr(tools, "_get_skills_container_path", lambda: "/mnt/skills")
(tmp_path / "workspace").mkdir(exist_ok=True)
assert view_root.is_dir()
@@ -326,6 +326,30 @@ def test_bash_runs_the_updated_skill_script(cache_setup, monkeypatch, tmp_path):
assert (view.root / "public" / "online" / "scripts" / "run.py").read_text(encoding="utf-8") == "print('two')\n"
def test_bash_records_a_direct_skill_script_invocation(cache_setup, monkeypatch, tmp_path):
root, _config, storage = cache_setup
script = root / "public" / "online" / "scripts"
script.mkdir()
(script / "run.py").write_text("print('used')\n", encoding="utf-8")
view = skill_view.refresh_skill_sandbox_view(storage=storage)
_tool_sandbox(tmp_path, view.root, monkeypatch)
recorded: list[tuple[str, object]] = []
monkeypatch.setattr(
tools,
"_record_skill_command",
lambda command, runtime: recorded.append((command, runtime)),
)
result = tools.bash_tool.func(
runtime=None,
description="test",
command="python /mnt/skills/public/online/scripts/run.py",
)
assert result == "ok"
assert recorded == [("python /mnt/skills/public/online/scripts/run.py", None)]
def test_bash_rejects_an_offline_skill(cache_setup, monkeypatch, tmp_path):
root, config, storage = cache_setup
_skill(root, "offline")
@@ -50,3 +50,21 @@ def test_canonical_write_file_is_still_recorded(monkeypatch) -> None:
middleware.wrap_tool_call(_request("write_file"), lambda _: _result("write_file"))
flush.assert_called_once()
def test_call_metrics_switch_skips_new_metric_rows(monkeypatch) -> None:
middleware = ToolMetricsMiddleware()
flush = Mock()
monkeypatch.setattr(middleware, "_flush", flush)
monkeypatch.setattr(
"deerflow.agents.middlewares.tool_metrics_middleware.get_call_metrics_settings",
lambda: SimpleNamespace(enabled=False),
)
result = middleware.wrap_tool_call(
_request("write_file"),
lambda _: _result("write_file"),
)
assert result.name == "write_file"
flush.assert_not_called()
@@ -20,8 +20,10 @@ from deerflow.config.app_config import AppConfig
BUILDERS = [build_lead_runtime_middlewares, build_subagent_runtime_middlewares]
def _config(enabled=None):
def _config(enabled=None, *, skill_enabled=None):
options = {} if enabled is None else {"tool_metrics_enabled": enabled}
if skill_enabled is not None:
options["skill_metrics_enabled"] = skill_enabled
return AppConfig(
sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"},
runtime_performance=options,
@@ -29,18 +31,29 @@ def _config(enabled=None):
@pytest.mark.parametrize("builder", BUILDERS)
@pytest.mark.parametrize("enabled", [None, False])
def test_paused_runtime_never_constructs_a_recorder(builder, enabled, monkeypatch):
def test_explicitly_paused_runtime_never_constructs_a_recorder(builder, monkeypatch):
constructor = Mock(side_effect=AssertionError("paused runtime constructed a DB recorder"))
monkeypatch.setattr(metrics, "ToolMetricsMiddleware", constructor)
middlewares = builder(app_config=_config(enabled))
middlewares = builder(app_config=_config(False, skill_enabled=False))
constructor.assert_not_called()
assert any(isinstance(m, ToolErrorHandlingMiddleware) for m in middlewares)
assert any(isinstance(m, LlmMetricsMiddleware) for m in middlewares)
@pytest.mark.parametrize("builder", BUILDERS)
def test_default_runtime_keeps_a_full_tool_and_skill_recorder(builder):
recorders = [
middleware
for middleware in builder(app_config=_config())
if isinstance(middleware, metrics.ToolMetricsMiddleware)
]
assert len(recorders) == 1
assert recorders[0]._skill_only is False
def _tool_wrappers(builder, enabled):
# Exercise the actual error/audit wrappers selected by both runtime builders.
return [m for m in builder(app_config=_config(enabled)) if isinstance(m, (ToolErrorHandlingMiddleware, metrics.ToolMetricsMiddleware))]
@@ -77,6 +90,30 @@ def test_sync_tool_execution_and_error_conversion_survive_pause(builder, enabled
assert flushed.call_args.args[0].status == result.status
@pytest.mark.parametrize("builder", BUILDERS)
def test_skill_only_mode_records_a_path_based_skill_call(builder, monkeypatch):
flushed = Mock()
monkeypatch.setattr(metrics.ToolMetricsMiddleware, "_flush", lambda _, record: flushed(record))
request = SimpleNamespace(
tool_call={
"id": "call-skill",
"name": "bash",
"args": {"command": "python /mnt/skills/public/report-maker/scripts/run.py"},
}
)
expected = ToolMessage(content="ok", tool_call_id="call-skill", name="bash")
def handler(_request):
return expected
for middleware in reversed(_tool_wrappers(builder, False)):
handler = partial(middleware.wrap_tool_call, handler=handler)
assert handler(request) is expected
assert flushed.call_count == 1
assert flushed.call_args.args[0].skill_name == "report-maker"
@pytest.mark.asyncio
@pytest.mark.parametrize("builder", BUILDERS)
@pytest.mark.parametrize("enabled", [False, True])