sync update from LAN repo (source dhk-1 @ c21a2a4a)
This commit is contained in:
1 parent
264e2200d6
commit
a1fa13a238
122 files changed
+8881
-623
No files matched your search
+71
-71
@@ -264,17 +264,17 @@ def _get_enabled_skills():
|
||||
def _get_enabled_skills_for_config(app_config: AppConfig | None = None) -> list[Skill]:
|
||||
"""Return enabled skills using the caller's config source.
|
||||
|
||||
Resolve storage from the caller's config and reuse the parsed catalogue only
|
||||
when its root matches. A cold cache or a different root falls back to that
|
||||
storage, keeping request-scoped skill directories isolated.
|
||||
Resolve storage from the caller's config and reuse the parsed catalogue only
|
||||
when its root matches. A cold cache or a different root falls back to that
|
||||
storage, keeping request-scoped skill directories isolated.
|
||||
"""
|
||||
if app_config is None:
|
||||
return _get_enabled_skills()
|
||||
storage = get_or_new_skill_storage(app_config=app_config)
|
||||
warm = get_warm_enabled_skills(storage)
|
||||
if warm is not None:
|
||||
return warm
|
||||
return _apply_always_on(_drop_archived_skills(list(storage.load_skills(enabled_only=True))))
|
||||
storage = get_or_new_skill_storage(app_config=app_config)
|
||||
warm = get_warm_enabled_skills(storage)
|
||||
if warm is not None:
|
||||
return warm
|
||||
return _apply_always_on(_drop_archived_skills(list(storage.load_skills(enabled_only=True))))
|
||||
|
||||
|
||||
def _skill_mutability_label(category: SkillCategory | str) -> str:
|
||||
@@ -617,44 +617,44 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p
|
||||
</subagent_system>"""
|
||||
|
||||
|
||||
_ORDINARY_QA_HEADING_RULES = {
|
||||
"hybrid_decimal": """- 一级章节使用 `## 一、标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
|
||||
- 二级章节按 `1.1`、`1.2`、`2.1` 的形式编号,三级及更深层级继续使用完整多级阿拉伯数字。
|
||||
- 确需更深层级时,四级章节使用 `##### 1.1.1.1 标题`,五级章节使用 `###### 1.1.1.1.1 标题`。""",
|
||||
"official_chinese": """- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。
|
||||
- 确需更深层级时,四级章节使用 `##### (1) 标题`,五级章节使用 `###### 1) 标题`。""",
|
||||
"decimal": """- 一级章节使用 `## 1 标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
|
||||
- 确需更深层级时,继续使用 `##### 1.1.1.1 标题`、`###### 1.1.1.1.1 标题`。""",
|
||||
"chapter_section": """- 一级章节使用 `## 第一章 标题`,二级章节使用 `### 第一节 标题`,三级章节使用 `#### 一、标题`。
|
||||
- 确需更深层级时,四级章节使用 `##### (一) 标题`,五级章节使用 `###### 1. 标题`。""",
|
||||
"none": """- 各级章节只写标题文字,不手写编号:`## 一级标题`、`### 二级标题`、`#### 三级标题`。
|
||||
- 确需更深层级时使用 `##### 四级标题`、`###### 五级标题`,仍不添加编号。""",
|
||||
}
|
||||
|
||||
|
||||
def _word_heading_numbering_style() -> str:
|
||||
"""Return the configured Word heading-numbering preset with a safe default."""
|
||||
try:
|
||||
return load_system_settings().prompt_prefix.word_heading_numbering_style
|
||||
except Exception:
|
||||
logger.debug("Failed to read the Word heading-numbering style; using default", exc_info=True)
|
||||
return "hybrid_decimal"
|
||||
|
||||
|
||||
def _build_ordinary_qa_markdown_format_section() -> str:
|
||||
rules = _ORDINARY_QA_HEADING_RULES.get(
|
||||
_word_heading_numbering_style(),
|
||||
_ORDINARY_QA_HEADING_RULES["hybrid_decimal"],
|
||||
)
|
||||
return f"""
|
||||
<ordinary_qa_markdown_format>
|
||||
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
|
||||
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
|
||||
{rules}
|
||||
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
|
||||
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
|
||||
</ordinary_qa_markdown_format>
|
||||
"""
|
||||
_ORDINARY_QA_HEADING_RULES = {
|
||||
"hybrid_decimal": """- 一级章节使用 `## 一、标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
|
||||
- 二级章节按 `1.1`、`1.2`、`2.1` 的形式编号,三级及更深层级继续使用完整多级阿拉伯数字。
|
||||
- 确需更深层级时,四级章节使用 `##### 1.1.1.1 标题`,五级章节使用 `###### 1.1.1.1.1 标题`。""",
|
||||
"official_chinese": """- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。
|
||||
- 确需更深层级时,四级章节使用 `##### (1) 标题`,五级章节使用 `###### 1) 标题`。""",
|
||||
"decimal": """- 一级章节使用 `## 1 标题`,二级章节使用 `### 1.1 标题`,三级章节使用 `#### 1.1.1 标题`。
|
||||
- 确需更深层级时,继续使用 `##### 1.1.1.1 标题`、`###### 1.1.1.1.1 标题`。""",
|
||||
"chapter_section": """- 一级章节使用 `## 第一章 标题`,二级章节使用 `### 第一节 标题`,三级章节使用 `#### 一、标题`。
|
||||
- 确需更深层级时,四级章节使用 `##### (一) 标题`,五级章节使用 `###### 1. 标题`。""",
|
||||
"none": """- 各级章节只写标题文字,不手写编号:`## 一级标题`、`### 二级标题`、`#### 三级标题`。
|
||||
- 确需更深层级时使用 `##### 四级标题`、`###### 五级标题`,仍不添加编号。""",
|
||||
}
|
||||
|
||||
|
||||
def _word_heading_numbering_style() -> str:
|
||||
"""Return the configured Word heading-numbering preset with a safe default."""
|
||||
try:
|
||||
return load_system_settings().prompt_prefix.word_heading_numbering_style
|
||||
except Exception:
|
||||
logger.debug("Failed to read the Word heading-numbering style; using default", exc_info=True)
|
||||
return "hybrid_decimal"
|
||||
|
||||
|
||||
def _build_ordinary_qa_markdown_format_section() -> str:
|
||||
rules = _ORDINARY_QA_HEADING_RULES.get(
|
||||
_word_heading_numbering_style(),
|
||||
_ORDINARY_QA_HEADING_RULES["hybrid_decimal"],
|
||||
)
|
||||
return f"""
|
||||
<ordinary_qa_markdown_format>
|
||||
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
|
||||
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
|
||||
{rules}
|
||||
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
|
||||
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
|
||||
</ordinary_qa_markdown_format>
|
||||
"""
|
||||
|
||||
|
||||
_ORDINARY_QA_VISUALIZATION_SECTION = """
|
||||
@@ -1261,13 +1261,13 @@ RAG/reference display mode is enabled.
|
||||
return prompt
|
||||
|
||||
|
||||
def _get_memory_context(
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
app_config: AppConfig | None = None,
|
||||
injection_enabled: bool = True,
|
||||
include_legacy_knowledge_routing: bool = True,
|
||||
) -> str:
|
||||
def _get_memory_context(
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
app_config: AppConfig | None = None,
|
||||
injection_enabled: bool = True,
|
||||
include_legacy_knowledge_routing: bool = True,
|
||||
) -> str:
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
|
||||
@@ -1278,10 +1278,10 @@ def _get_memory_context(
|
||||
agent_name: agent 标识;``None`` 时按 "default" 处理。
|
||||
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
|
||||
injection_enabled: 运行时开关;``False`` 时跳过注入直接返回空串。
|
||||
include_legacy_knowledge_routing: 是否注入 ``/mnt/knowledge/routing.md``。
|
||||
智能普通知识库检索已有数据库授权范围和专用检索工具时必须关闭,
|
||||
避免旧文件路由把“这个库”错误引向沙箱目录;其它记忆 provider
|
||||
仍按原配置保留。
|
||||
include_legacy_knowledge_routing: 是否注入 ``/mnt/knowledge/routing.md``。
|
||||
智能普通知识库检索已有数据库授权范围和专用检索工具时必须关闭,
|
||||
避免旧文件路由把“这个库”错误引向沙箱目录;其它记忆 provider
|
||||
仍按原配置保留。
|
||||
|
||||
Returns:
|
||||
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
|
||||
@@ -1306,11 +1306,11 @@ def _get_memory_context(
|
||||
)
|
||||
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
|
||||
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
|
||||
context_content = manager.build_system_prompt(
|
||||
excluded_provider_names=(
|
||||
None if include_legacy_knowledge_routing else {"knowledge"}
|
||||
)
|
||||
)
|
||||
context_content = manager.build_system_prompt(
|
||||
excluded_provider_names=(
|
||||
None if include_legacy_knowledge_routing else {"knowledge"}
|
||||
)
|
||||
)
|
||||
|
||||
if not context_content.strip():
|
||||
return ""
|
||||
@@ -1878,7 +1878,7 @@ def apply_prompt_template(
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
include_legacy_knowledge_routing: bool = True,
|
||||
include_legacy_knowledge_routing: bool = True,
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
@@ -1887,12 +1887,12 @@ def apply_prompt_template(
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
memory_context = _get_memory_context(
|
||||
storage_agent_id,
|
||||
app_config=app_config,
|
||||
injection_enabled=memory_injection_enabled,
|
||||
include_legacy_knowledge_routing=include_legacy_knowledge_routing,
|
||||
)
|
||||
memory_context = _get_memory_context(
|
||||
storage_agent_id,
|
||||
app_config=app_config,
|
||||
injection_enabled=memory_injection_enabled,
|
||||
include_legacy_knowledge_routing=include_legacy_knowledge_routing,
|
||||
)
|
||||
|
||||
# Include subagent section only if enabled (from runtime parameter)
|
||||
n = max_concurrent_subagents
|
||||
@@ -2004,8 +2004,8 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
ordinary_qa_format_section = (
|
||||
_build_ordinary_qa_markdown_format_section()
|
||||
ordinary_qa_format_section = (
|
||||
_build_ordinary_qa_markdown_format_section()
|
||||
if _ordinary_qa_markdown_format_enabled()
|
||||
and not is_bootstrap
|
||||
and not writing_mode
|
||||
|
||||
+29
-24
@@ -136,13 +136,18 @@ def _build_runtime_middlewares(
|
||||
# wrap_tool_call chain: its handler is the raw tool, so it observes the
|
||||
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
|
||||
# an error ToolMessage) and the raw return value.
|
||||
# Omit the recorder entirely while paused: no record allocation, skill-merge
|
||||
# lookup, or fire-and-forget DB task is created by any tool invocation.
|
||||
# Keep tool execution/error handling and LLM metrics independent of this flag.
|
||||
if app_config.runtime_performance.tool_metrics_enabled:
|
||||
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
|
||||
|
||||
middlewares.append(ToolMetricsMiddleware())
|
||||
# Full auditing can stay paused under high concurrency while the low-volume
|
||||
# skill-only recorder remains active. In skill-only mode non-skill calls are
|
||||
# passed through without a merge query or fire-and-forget DB write.
|
||||
performance = app_config.runtime_performance
|
||||
if performance.tool_metrics_enabled or performance.skill_metrics_enabled:
|
||||
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
|
||||
|
||||
middlewares.append(
|
||||
ToolMetricsMiddleware(
|
||||
skill_only=not performance.tool_metrics_enabled,
|
||||
)
|
||||
)
|
||||
return middlewares
|
||||
|
||||
|
||||
@@ -179,20 +184,20 @@ def build_subagent_runtime_middlewares(
|
||||
model_name = app_config.models[0].name
|
||||
|
||||
model_config = app_config.get_model_config(model_name) if model_name else None
|
||||
if model_config is not None and model_config.supports_vision:
|
||||
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
|
||||
|
||||
middlewares.append(ViewImageMiddleware())
|
||||
|
||||
# Must remain the innermost model-call wrapper: subagent-specific context,
|
||||
# vision payloads, system prompts and tool schemas all need to be included
|
||||
# in the final input-size decision.
|
||||
from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware
|
||||
|
||||
middlewares.append(
|
||||
ContextWindowGuardMiddleware(
|
||||
max_input_tokens=model_config.max_input_tokens if model_config is not None else None,
|
||||
)
|
||||
)
|
||||
|
||||
return middlewares
|
||||
if model_config is not None and model_config.supports_vision:
|
||||
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
|
||||
|
||||
middlewares.append(ViewImageMiddleware())
|
||||
|
||||
# Must remain the innermost model-call wrapper: subagent-specific context,
|
||||
# vision payloads, system prompts and tool schemas all need to be included
|
||||
# in the final input-size decision.
|
||||
from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware
|
||||
|
||||
middlewares.append(
|
||||
ContextWindowGuardMiddleware(
|
||||
max_input_tokens=model_config.max_input_tokens if model_config is not None else None,
|
||||
)
|
||||
)
|
||||
|
||||
return middlewares
|
||||
+28
-5
@@ -1,6 +1,7 @@
|
||||
"""Middleware that records per-tool-call metrics for the admin audit page.
|
||||
|
||||
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
|
||||
While the administrator-controlled call-metrics switch is enabled, every tool
|
||||
/ skill invocation ends with one row in ``tool_call_metrics``:
|
||||
- Successes record the wall-clock duration.
|
||||
- Failures record a coarse ``error_category`` plus a one-line message.
|
||||
|
||||
@@ -40,6 +41,7 @@ from langgraph.errors import GraphBubbleUp
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.config.system_settings import get_call_metrics_settings
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write, get_session_factory
|
||||
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
|
||||
@@ -188,11 +190,18 @@ def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | Non
|
||||
|
||||
|
||||
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Wrap each tool call to time it and persist a metric row."""
|
||||
"""Time tool calls, optionally persisting only attributable skill calls."""
|
||||
|
||||
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
store: ToolMetricsStore | None = None,
|
||||
skill_only: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._store_override = store
|
||||
self._skill_only = skill_only
|
||||
|
||||
# -- store / context ----------------------------------------------------
|
||||
|
||||
@@ -205,6 +214,12 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
return None
|
||||
return make_tool_metrics_store(sf)
|
||||
|
||||
@staticmethod
|
||||
def _metrics_enabled() -> bool:
|
||||
"""Read the administrator switch at call time so no restart is needed."""
|
||||
|
||||
return bool(get_call_metrics_settings().enabled)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
@@ -280,7 +295,11 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
if is_internal_tool_name(record.tool_name):
|
||||
if (
|
||||
not self._metrics_enabled()
|
||||
or is_internal_tool_name(record.tool_name)
|
||||
or (self._skill_only and not record.skill_name)
|
||||
):
|
||||
return handler(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
@@ -303,7 +322,11 @@ class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
if is_internal_tool_name(record.tool_name):
|
||||
if (
|
||||
not self._metrics_enabled()
|
||||
or is_internal_tool_name(record.tool_name)
|
||||
or (self._skill_only and not record.skill_name)
|
||||
):
|
||||
return await handler(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
|
||||
@@ -482,13 +482,14 @@ def _load_and_cache_app_config(config_path: str | None = None) -> AppConfig:
|
||||
)
|
||||
performance = _app_config.runtime_performance
|
||||
logger.info(
|
||||
"RUNTIME_PERFORMANCE skill_view_reuse_enabled=%s skill_view_cache_enabled=%s model_timing_nonblocking_enabled=%s asyncio_thread_pool_enabled=%s asyncio_thread_pool_size=%d tool_metrics_enabled=%s",
|
||||
"RUNTIME_PERFORMANCE skill_view_reuse_enabled=%s skill_view_cache_enabled=%s model_timing_nonblocking_enabled=%s asyncio_thread_pool_enabled=%s asyncio_thread_pool_size=%d tool_metrics_enabled=%s skill_metrics_enabled=%s",
|
||||
performance.skill_view_reuse_enabled,
|
||||
performance.skill_view_cache_enabled,
|
||||
performance.model_timing_nonblocking_enabled,
|
||||
performance.asyncio_thread_pool_enabled,
|
||||
performance.asyncio_thread_pool_size,
|
||||
performance.tool_metrics_enabled,
|
||||
performance.skill_metrics_enabled,
|
||||
)
|
||||
return _app_config
|
||||
|
||||
|
||||
+19
-5
@@ -2,9 +2,11 @@
|
||||
|
||||
Skill-view reuse and cache are on when those keys are absent. Set them to
|
||||
false in config.yaml to restore a full refresh on every tool call. Model
|
||||
timing and the Gateway thread pool stay off unless configured. If application
|
||||
configuration cannot be loaded, every switch stays off. Tool-call metric writes
|
||||
are paused by default; explicitly enable them to resume audit statistics.
|
||||
timing and the Gateway thread pool stay off unless configured. Tool and skill
|
||||
metrics are on by default so the management leaderboard has a complete source
|
||||
of truth. Deployments that need to shed this best-effort audit write load can
|
||||
explicitly pause either metric type in ``config.yaml``. If configuration
|
||||
cannot be loaded, the fallback keeps every switch off.
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
@@ -15,7 +17,14 @@ class RuntimePerformanceConfig(BaseModel):
|
||||
|
||||
skill_view_reuse_enabled: bool = Field(default=True, description="Reuse one verified skill view within each sandbox tool invocation. Set false to refresh on every call.")
|
||||
skill_view_cache_enabled: bool = Field(default=True, description="Reuse unchanged skill views after validating source, target and availability versions. Set false to rebuild every call.")
|
||||
tool_metrics_enabled: bool = Field(default=False, description="Persist tool/skill audit statistics to tool_call_metrics. Disabled by default to avoid competing with interactive database traffic; historical reads remain available.")
|
||||
tool_metrics_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Persist tool/skill audit statistics to tool_call_metrics. Enabled by default so management leaderboard tool rankings receive data; set false only to pause ordinary tool auditing.",
|
||||
)
|
||||
skill_metrics_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Persist calls that can be attributed to a concrete skill, keeping skill statistics available even if ordinary tool auditing is paused.",
|
||||
)
|
||||
model_timing_nonblocking_enabled: bool = Field(default=False, description="Dispatch model timing logs through a dedicated bounded queue, outside the shared tool executor")
|
||||
asyncio_thread_pool_enabled: bool = Field(default=False, description="Override the Gateway event loop's default executor at startup; disabled preserves Python defaults")
|
||||
asyncio_thread_pool_size: int = Field(default=100, ge=1, le=1024, strict=True, description="Maximum Gateway executor threads when enabled, unless overridden by the documented environment variables")
|
||||
@@ -24,7 +33,12 @@ class RuntimePerformanceConfig(BaseModel):
|
||||
# Field defaults turn the skill-view fast path on. This fallback is only for
|
||||
# callers that cannot read application configuration, and it keeps every
|
||||
# switch off.
|
||||
_DEFAULT = RuntimePerformanceConfig(skill_view_reuse_enabled=False, skill_view_cache_enabled=False)
|
||||
_DEFAULT = RuntimePerformanceConfig(
|
||||
skill_view_reuse_enabled=False,
|
||||
skill_view_cache_enabled=False,
|
||||
tool_metrics_enabled=False,
|
||||
skill_metrics_enabled=False,
|
||||
)
|
||||
|
||||
|
||||
def get_runtime_performance_config() -> RuntimePerformanceConfig:
|
||||
|
||||
+264
-1
@@ -21,7 +21,7 @@ from pathlib import Path
|
||||
from typing import Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
@@ -474,6 +474,20 @@ class SkillCompressionSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SkillVersioningSettings(BaseModel):
|
||||
"""Deployment-wide gate for custom-skill package version history.
|
||||
|
||||
The default is intentionally disabled so existing deployments retain the
|
||||
legacy in-place editing behaviour until an administrator opts in.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether custom-skill package versioning and switching are enabled",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class AgentModuleSettings(BaseModel):
|
||||
"""Admin-configured「按类型聚合」module → selected agent-tag ids.
|
||||
|
||||
@@ -668,6 +682,24 @@ class WriteFileCompatibilitySettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class CallMetricsSettings(BaseModel):
|
||||
"""Administrator-controlled recording for tool and skill call statistics.
|
||||
|
||||
Both rankings use the same ``tool_call_metrics`` audit stream. Keeping a
|
||||
single switch prevents a partially enabled configuration where the
|
||||
management page can show one ranking but not the other.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Whether new tool and skill calls are persisted for management "
|
||||
"statistics. Existing records are retained when disabled."
|
||||
),
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class RetryKeepPartialSettings(BaseModel):
|
||||
"""Keep already-streamed partial answers visible across silent LLM retries.
|
||||
|
||||
@@ -717,6 +749,15 @@ class RoundtableRoomLimitSettings(BaseModel):
|
||||
"but never extend this deadline."
|
||||
),
|
||||
)
|
||||
lease_minutes: int = Field(
|
||||
default=5,
|
||||
ge=1,
|
||||
le=60,
|
||||
description=(
|
||||
"How long a foreground step-2 lease stays occupied without a renewal. "
|
||||
"The browser heartbeat renews it; this does not extend max_hold_minutes."
|
||||
),
|
||||
)
|
||||
max_cycles: int = Field(
|
||||
default=8,
|
||||
ge=2,
|
||||
@@ -769,7 +810,40 @@ class HtmlMapSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
_GENERATED_PAGE_LOGO_FILE_RE = re.compile(r"^logo-[a-f0-9]{32}\.(?:png|jpg|webp)$")
|
||||
|
||||
|
||||
class GeneratedPageBrandingSettings(BaseModel):
|
||||
"""Optional generated-page logo settings retained for offline bundles."""
|
||||
|
||||
chrome_enabled: bool = True
|
||||
header_layout: Literal["left", "center"] = "center"
|
||||
system_name: str | None = Field(default=None, max_length=80)
|
||||
footer_text: str | None = Field(default=None, max_length=160)
|
||||
logo_filename: str | None = None
|
||||
logo_mode: Literal["custom", "default", "none"] = "custom"
|
||||
logo_media_type: Literal["image/png", "image/jpeg", "image/webp"] | None = None
|
||||
logo_version: str | None = Field(default=None, max_length=64)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("logo_filename")
|
||||
@classmethod
|
||||
def _validate_logo_filename(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().lower()
|
||||
if not _GENERATED_PAGE_LOGO_FILE_RE.fullmatch(normalized):
|
||||
raise ValueError("Invalid generated-page logo filename")
|
||||
return normalized
|
||||
|
||||
|
||||
_APP_IDENTITY_LOGO_FILE_RE = re.compile(r"^logo-[a-f0-9]{32}\.(?:png|jpg|webp)$")
|
||||
_APP_IDENTITY_LIGHT_RED_HEADER_FILE_RE = re.compile(
|
||||
r"^light-red-header-[a-f0-9]{32}\.(?:png|jpg|webp|svg)$"
|
||||
)
|
||||
_APP_IDENTITY_LIGHT_RED_ACTIVE_FILE_RE = re.compile(
|
||||
r"^light-red-active-[a-f0-9]{32}\.(?:png|jpg|webp|svg)$"
|
||||
)
|
||||
|
||||
|
||||
class AppIdentitySettings(BaseModel):
|
||||
@@ -780,9 +854,21 @@ class AppIdentitySettings(BaseModel):
|
||||
"""
|
||||
|
||||
system_name: str = Field(default="", max_length=80)
|
||||
header_title_font_mode: Literal["adaptive", "fixed"] = "adaptive"
|
||||
header_title_font_size: int = Field(default=30, ge=16, le=40)
|
||||
logo_filename: str | None = None
|
||||
logo_media_type: Literal["image/png", "image/jpeg", "image/webp"] | None = None
|
||||
logo_version: str | None = None
|
||||
light_red_header_background_filename: str | None = None
|
||||
light_red_header_background_media_type: Literal[
|
||||
"image/png", "image/jpeg", "image/webp", "image/svg+xml"
|
||||
] | None = None
|
||||
light_red_header_background_version: str | None = None
|
||||
light_red_nav_active_filename: str | None = None
|
||||
light_red_nav_active_media_type: Literal[
|
||||
"image/png", "image/jpeg", "image/webp", "image/svg+xml"
|
||||
] | None = None
|
||||
light_red_nav_active_version: str | None = None
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("system_name")
|
||||
@@ -801,6 +887,26 @@ class AppIdentitySettings(BaseModel):
|
||||
raise ValueError("Invalid application logo filename")
|
||||
return normalized
|
||||
|
||||
@field_validator("light_red_header_background_filename")
|
||||
@classmethod
|
||||
def _validate_light_red_header_filename(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().lower()
|
||||
if not _APP_IDENTITY_LIGHT_RED_HEADER_FILE_RE.fullmatch(normalized):
|
||||
raise ValueError("Invalid light-red header background filename")
|
||||
return normalized
|
||||
|
||||
@field_validator("light_red_nav_active_filename")
|
||||
@classmethod
|
||||
def _validate_light_red_active_filename(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().lower()
|
||||
if not _APP_IDENTITY_LIGHT_RED_ACTIVE_FILE_RE.fullmatch(normalized):
|
||||
raise ValueError("Invalid light-red active navigation filename")
|
||||
return normalized
|
||||
|
||||
|
||||
def app_identity_logo_path(
|
||||
settings: AppIdentitySettings | None = None,
|
||||
@@ -814,6 +920,42 @@ def app_identity_logo_path(
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
def app_identity_light_red_header_background_path(
|
||||
settings: AppIdentitySettings | None = None,
|
||||
) -> Path | None:
|
||||
"""Resolve the uploaded light-red header background inside runtime storage."""
|
||||
|
||||
current = settings or load_system_settings().app_identity
|
||||
if not current.light_red_header_background_filename:
|
||||
return None
|
||||
path = runtime_home() / "app-identity" / current.light_red_header_background_filename
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
def app_identity_light_red_nav_active_path(
|
||||
settings: AppIdentitySettings | None = None,
|
||||
) -> Path | None:
|
||||
"""Resolve the uploaded light-red active-navigation image inside runtime storage."""
|
||||
|
||||
current = settings or load_system_settings().app_identity
|
||||
if not current.light_red_nav_active_filename:
|
||||
return None
|
||||
path = runtime_home() / "app-identity" / current.light_red_nav_active_filename
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
def generated_page_logo_path(
|
||||
settings: GeneratedPageBrandingSettings | None = None,
|
||||
) -> Path | None:
|
||||
"""Resolve the custom generated-page logo without accepting arbitrary paths."""
|
||||
|
||||
current = settings or load_system_settings().generated_page_branding
|
||||
if current.logo_mode != "custom" or not current.logo_filename:
|
||||
return None
|
||||
path = runtime_home() / "generated-page-branding" / current.logo_filename
|
||||
return path if path.is_file() else None
|
||||
|
||||
|
||||
class HtmlRuntimeSettings(BaseModel):
|
||||
"""Addresses written into newly generated HTML pages.
|
||||
|
||||
@@ -844,6 +986,108 @@ class StrategyProjectSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class GeneralQaPageDescription(BaseModel):
|
||||
"""Administrator-provided copy for the ordinary Q&A landing page."""
|
||||
|
||||
welcome_title: str = Field(default="", max_length=80)
|
||||
welcome_description: str = Field(default="", max_length=500)
|
||||
input_placeholder: str = Field(default="", max_length=120)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("welcome_title", "welcome_description", "input_placeholder")
|
||||
@classmethod
|
||||
def _normalize_copy(cls, value: str | None) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
class GeneralQaPageDescriptions(BaseModel):
|
||||
"""Separate ordinary-Q&A copy for the full and compact layouts."""
|
||||
|
||||
full: GeneralQaPageDescription = Field(default_factory=GeneralQaPageDescription)
|
||||
compact: GeneralQaPageDescription = Field(default_factory=GeneralQaPageDescription)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _migrate_legacy_single_description(cls, value: object) -> object:
|
||||
if isinstance(value, GeneralQaPageDescription):
|
||||
return {"full": value, "compact": value}
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
if "full" in value or "compact" in value:
|
||||
return value
|
||||
legacy = {
|
||||
key: value.get(key, "")
|
||||
for key in ("welcome_title", "welcome_description", "input_placeholder")
|
||||
}
|
||||
return {"full": legacy, "compact": legacy}
|
||||
|
||||
|
||||
class PageHeaderDescription(BaseModel):
|
||||
"""Configurable title and explanatory copy for a product landing page."""
|
||||
|
||||
title: str = Field(default="", max_length=80)
|
||||
description: str = Field(default="", max_length=500)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("title", "description")
|
||||
@classmethod
|
||||
def _normalize_copy(cls, value: str | None) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
class AIWritingGuideCard(BaseModel):
|
||||
"""Editable text for one of the fixed AI-writing welcome cards."""
|
||||
|
||||
title: str = Field(default="", max_length=80)
|
||||
description: str = Field(default="", max_length=240)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("title", "description")
|
||||
@classmethod
|
||||
def _normalize_copy(cls, value: str | None) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _empty_ai_writing_guide_cards() -> list[AIWritingGuideCard]:
|
||||
return [AIWritingGuideCard() for _ in range(3)]
|
||||
|
||||
|
||||
class AIWritingPageDescription(PageHeaderDescription):
|
||||
"""Configurable copy for the classic AI-writing welcome page."""
|
||||
|
||||
guide_cards: list[AIWritingGuideCard] = Field(
|
||||
default_factory=_empty_ai_writing_guide_cards,
|
||||
max_length=3,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _keep_three_guide_cards(cls, value: object) -> object:
|
||||
if not isinstance(value, dict) or not isinstance(value.get("guide_cards"), list):
|
||||
return value
|
||||
cards = value["guide_cards"][:3]
|
||||
return {**value, "guide_cards": [*cards, *({} for _ in range(3 - len(cards)))]}
|
||||
|
||||
|
||||
class PageDescriptionsSettings(BaseModel):
|
||||
"""Page-scoped, administrator-managed display copy."""
|
||||
|
||||
general_qa: GeneralQaPageDescriptions = Field(default_factory=GeneralQaPageDescriptions)
|
||||
knowledge_base: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
ai_writing: AIWritingPageDescription = Field(default_factory=AIWritingPageDescription)
|
||||
strategy_projects: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
deep_research: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
gongwen: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
light_app_management: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
light_app_list: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
agents: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
business_chains: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
skills: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
knowledge_management: PageHeaderDescription = Field(default_factory=PageHeaderDescription)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class ModelPromptSuffix(BaseModel):
|
||||
"""Administrator-owned instructions appended to one configured model's system prompt."""
|
||||
|
||||
@@ -862,9 +1106,14 @@ class SystemSettings(BaseModel):
|
||||
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
|
||||
publish_squares: PublishSquaresSettings = Field(default_factory=PublishSquaresSettings)
|
||||
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
|
||||
generated_page_branding: GeneratedPageBrandingSettings = Field(
|
||||
default_factory=GeneratedPageBrandingSettings
|
||||
)
|
||||
page_descriptions: PageDescriptionsSettings = Field(default_factory=PageDescriptionsSettings)
|
||||
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
|
||||
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
|
||||
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
|
||||
skill_versioning: SkillVersioningSettings = Field(default_factory=SkillVersioningSettings)
|
||||
plugin_management: PluginManagementSettings = Field(default_factory=PluginManagementSettings)
|
||||
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
|
||||
wx_common_agents: WxCommonAgentsSettings = Field(default_factory=WxCommonAgentsSettings)
|
||||
@@ -875,6 +1124,7 @@ class SystemSettings(BaseModel):
|
||||
write_file_compatibility: WriteFileCompatibilitySettings = Field(
|
||||
default_factory=WriteFileCompatibilitySettings
|
||||
)
|
||||
call_metrics: CallMetricsSettings = Field(default_factory=CallMetricsSettings)
|
||||
retry_keep_partial: RetryKeepPartialSettings = Field(
|
||||
default_factory=RetryKeepPartialSettings
|
||||
)
|
||||
@@ -1007,6 +1257,19 @@ def get_write_file_compatibility_settings() -> WriteFileCompatibilitySettings:
|
||||
return WriteFileCompatibilitySettings()
|
||||
|
||||
|
||||
def get_call_metrics_settings() -> CallMetricsSettings:
|
||||
"""Return the live call-metrics switch, defaulting to recording enabled."""
|
||||
|
||||
try:
|
||||
return load_system_settings().call_metrics
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to read call metrics setting; keeping metrics enabled",
|
||||
exc_info=True,
|
||||
)
|
||||
return CallMetricsSettings()
|
||||
|
||||
|
||||
def get_roundtable_room_limit_settings() -> RoundtableRoomLimitSettings:
|
||||
"""Return the admin-controlled roundtable room cap, defaulting to 3 enabled rooms."""
|
||||
try:
|
||||
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
"""Build portable ZIP bundles for generated HTML artifacts.
|
||||
|
||||
Generated pages intentionally reference the Gateway's vendored browser runtime
|
||||
instead of public CDNs. A plain ``.html`` download therefore stops working
|
||||
after it is moved away from the DeerFlow deployment. This module rewrites
|
||||
those trusted Gateway URLs to relative paths and copies the corresponding
|
||||
packaged assets into a ZIP. It never downloads arbitrary third-party URLs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from importlib.resources import files
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from deerflow.html_runtime.public_images import resolve_html_image_path, verify_html_image_token
|
||||
|
||||
_STATIC_ASSETS = {
|
||||
"alpine.min.js",
|
||||
"bootstrap.bundle.min.js",
|
||||
"bootstrap.min.css",
|
||||
"brand-logo-dark.png",
|
||||
"echarts.min.js",
|
||||
"g6.min.js",
|
||||
"leaflet.css",
|
||||
"leaflet.js",
|
||||
"topbar-dark.svg",
|
||||
"topbar-light.svg",
|
||||
}
|
||||
_GEO_MAP_ASSETS = {
|
||||
"china-provinces.geojson",
|
||||
"japan-prefectures.geojson",
|
||||
"taiwan-counties.geojson",
|
||||
"us-states.geojson",
|
||||
}
|
||||
_COGNITIVE_ASSET_SUFFIXES = {
|
||||
".css",
|
||||
".js",
|
||||
".json",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".mjs",
|
||||
".png",
|
||||
".svg",
|
||||
".woff2",
|
||||
}
|
||||
_LICENSE_BY_ASSET = {
|
||||
"alpine.min.js": "LICENSE.alpine.txt",
|
||||
"bootstrap.bundle.min.js": "LICENSE.bootstrap.txt",
|
||||
"bootstrap.min.css": "LICENSE.bootstrap.txt",
|
||||
"echarts.min.js": "LICENSE.echarts.txt",
|
||||
"g6.min.js": "LICENSE.g6.txt",
|
||||
"leaflet.css": "LICENSE.leaflet.txt",
|
||||
"leaflet.js": "LICENSE.leaflet.txt",
|
||||
}
|
||||
_RUNTIME_ARCHIVE_ROOT = PurePosixPath("assets/html-runtime")
|
||||
_HTML_IMAGES_ARCHIVE_ROOT = PurePosixPath("assets/html-images")
|
||||
_STANDARD_RUNTIME_BASE_RE = (
|
||||
r"(?:https?://[^/\"'\s<>()]+)?"
|
||||
r"(?:/[A-Za-z0-9._~!$&()*+,;=:@%+-]+)*/api/public/html-runtime"
|
||||
)
|
||||
_HTML_IMAGE_URL_RE = re.compile(
|
||||
r"(?P<url>(?:https?://[^/\"'\s<>()]+)?"
|
||||
r"(?:/[A-Za-z0-9._~!$&()*+,;=:@%+-]+)*/api/public/html-images/"
|
||||
r"(?P<token>[^\"'\s<>()?#]+)(?:\?[^\"'\s<>()#]*)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_EXTERNAL_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OfflineBundleResult:
|
||||
"""Summary of one completed offline bundle."""
|
||||
|
||||
bundled_assets: tuple[str, ...]
|
||||
external_urls: tuple[str, ...]
|
||||
|
||||
|
||||
def _runtime_base_pattern(runtime_base_url: str) -> str:
|
||||
bases = [_STANDARD_RUNTIME_BASE_RE]
|
||||
configured = str(runtime_base_url or "").strip().rstrip("/")
|
||||
if configured:
|
||||
bases.insert(0, re.escape(configured))
|
||||
return "(?:" + "|".join(bases) + ")"
|
||||
|
||||
|
||||
def _safe_runtime_source(runtime_dir: Path, relative_path: str) -> Path | None:
|
||||
"""Resolve only assets that the public HTML-runtime router exposes."""
|
||||
|
||||
normalized = relative_path.strip().lstrip("/").replace("\\", "/")
|
||||
pure_path = PurePosixPath(normalized)
|
||||
if not normalized or pure_path.is_absolute() or ".." in pure_path.parts:
|
||||
return None
|
||||
|
||||
allowed = False
|
||||
if len(pure_path.parts) == 1 and normalized in _STATIC_ASSETS:
|
||||
allowed = True
|
||||
elif len(pure_path.parts) == 2 and pure_path.parts[0] == "maps" and pure_path.name in _GEO_MAP_ASSETS:
|
||||
allowed = True
|
||||
elif pure_path.parts and pure_path.parts[0] == "cognitive-situation-map":
|
||||
allowed = pure_path.suffix.lower() in _COGNITIVE_ASSET_SUFFIXES
|
||||
if not allowed:
|
||||
return None
|
||||
|
||||
root = runtime_dir.resolve()
|
||||
source = (root / Path(*pure_path.parts)).resolve()
|
||||
try:
|
||||
source.relative_to(root)
|
||||
except ValueError:
|
||||
return None
|
||||
return source if source.is_file() else None
|
||||
|
||||
|
||||
def _add_file(
|
||||
archive_files: dict[str, Path],
|
||||
archive_path: PurePosixPath,
|
||||
source: Path,
|
||||
) -> None:
|
||||
archive_files[archive_path.as_posix()] = source
|
||||
|
||||
|
||||
def _add_cognitive_runtime(archive_files: dict[str, Path], runtime_dir: Path) -> None:
|
||||
source_root = runtime_dir / "cognitive-situation-map"
|
||||
if not source_root.is_dir():
|
||||
return
|
||||
for source in source_root.rglob("*"):
|
||||
if not source.is_file():
|
||||
continue
|
||||
is_license = source.name.lower().startswith("license")
|
||||
if source.suffix.lower() not in _COGNITIVE_ASSET_SUFFIXES and not is_license:
|
||||
continue
|
||||
relative = source.relative_to(runtime_dir)
|
||||
_add_file(archive_files, _RUNTIME_ARCHIVE_ROOT / PurePosixPath(relative.as_posix()), source)
|
||||
|
||||
|
||||
def _generated_logo_destination(generated_logo_path: Path | None) -> PurePosixPath:
|
||||
suffix = generated_logo_path.suffix.lower() if generated_logo_path is not None else ".png"
|
||||
if suffix not in {".jpg", ".jpeg", ".png", ".webp"}:
|
||||
suffix = ".png"
|
||||
return _RUNTIME_ARCHIVE_ROOT / f"generated-page-logo{suffix}"
|
||||
|
||||
|
||||
def _runtime_script(
|
||||
runtime_dir: Path,
|
||||
*,
|
||||
map_tile_url: str,
|
||||
include_geo_maps: bool,
|
||||
) -> str:
|
||||
config: dict[str, object] = {"geoMaps": {}, "mapTileUrl": map_tile_url}
|
||||
if include_geo_maps:
|
||||
config["geoMaps"] = {
|
||||
Path(filename).stem: (_RUNTIME_ARCHIVE_ROOT / "maps" / filename).as_posix()
|
||||
for filename in sorted(_GEO_MAP_ASSETS)
|
||||
}
|
||||
prefix = "window.DEERFLOW_HTML_RUNTIME_CONFIG=" + json.dumps(
|
||||
config,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
) + ";\n"
|
||||
return prefix + (runtime_dir / "deerflow-runtime.js").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _readme(original_name: str, external_urls: list[str]) -> str:
|
||||
external_note = (
|
||||
"未打包的外部地址已记录在 manifest.json 的 external_urls 中;例如在线地图瓦片、"
|
||||
"用户主动引用的第三方图片或接口仍需要网络。"
|
||||
if external_urls
|
||||
else "本页没有检测到仍需联网的 http(s) 地址。"
|
||||
)
|
||||
return (
|
||||
"DeerFlow 网页离线包\n"
|
||||
"====================\n\n"
|
||||
f"原始文件:{original_name}\n"
|
||||
"入口文件:index.html\n"
|
||||
"依赖目录:assets/\n\n"
|
||||
"部署方式:请保持目录结构不变,将解压后的全部文件一起放到任意静态网站目录。\n"
|
||||
"若页面含 GeoJSON 地图等 fetch 请求,不建议直接双击 file:// 打开;可使用 Nginx、"
|
||||
"Apache、IIS 或其他静态 HTTP 服务访问 index.html。\n\n"
|
||||
f"{external_note}\n"
|
||||
)
|
||||
|
||||
|
||||
def build_html_offline_bundle(
|
||||
html_path: Path,
|
||||
destination: Path,
|
||||
*,
|
||||
runtime_base_url: str,
|
||||
map_tile_url: str,
|
||||
generated_logo_path: Path | None = None,
|
||||
runtime_dir: Path | None = None,
|
||||
) -> OfflineBundleResult:
|
||||
"""Write a portable HTML ZIP to *destination* and return its manifest summary."""
|
||||
|
||||
html_path = Path(html_path)
|
||||
destination = Path(destination)
|
||||
runtime_dir = runtime_dir or Path(str(files("deerflow.html_runtime.assets")))
|
||||
runtime_dir = Path(runtime_dir)
|
||||
html = html_path.read_text(encoding="utf-8", errors="replace")
|
||||
archive_files: dict[str, Path] = {}
|
||||
generated_files: dict[str, str] = {}
|
||||
bundled_assets: set[str] = set()
|
||||
licenses: set[str] = set()
|
||||
include_geo_maps = bool(re.search(r"data-df-geo\s*=", html, re.IGNORECASE))
|
||||
uses_leaflet_map = bool(re.search(r"data-df-map\s*=", html, re.IGNORECASE))
|
||||
|
||||
base_pattern = _runtime_base_pattern(runtime_base_url)
|
||||
cognitive_base_re = re.compile(
|
||||
rf"(?P<url>{base_pattern}/cognitive-situation-map)(?=$|[\"'\s<>()/])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def replace_cognitive_base(match: re.Match[str]) -> str:
|
||||
_add_cognitive_runtime(archive_files, runtime_dir)
|
||||
bundled_assets.add((_RUNTIME_ARCHIVE_ROOT / "cognitive-situation-map").as_posix() + "/")
|
||||
return (_RUNTIME_ARCHIVE_ROOT / "cognitive-situation-map").as_posix()
|
||||
|
||||
html = cognitive_base_re.sub(replace_cognitive_base, html)
|
||||
|
||||
runtime_url_re = re.compile(
|
||||
rf"(?P<url>{base_pattern}/(?P<asset>[A-Za-z0-9._/-]+)"
|
||||
r"(?:\?[^\"'\s<>()#]*)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def replace_runtime_url(match: re.Match[str]) -> str:
|
||||
asset = match.group("asset").strip("/")
|
||||
if asset == "generated-page-logo":
|
||||
source = generated_logo_path if generated_logo_path and generated_logo_path.is_file() else runtime_dir / "brand-logo-dark.png"
|
||||
if not source.is_file():
|
||||
return match.group("url")
|
||||
archive_path = _generated_logo_destination(source)
|
||||
_add_file(archive_files, archive_path, source)
|
||||
bundled_assets.add(archive_path.as_posix())
|
||||
return archive_path.as_posix()
|
||||
if asset == "deerflow-runtime.js":
|
||||
source = runtime_dir / asset
|
||||
if not source.is_file():
|
||||
return match.group("url")
|
||||
archive_path = _RUNTIME_ARCHIVE_ROOT / asset
|
||||
generated_files[archive_path.as_posix()] = _runtime_script(
|
||||
runtime_dir,
|
||||
map_tile_url=map_tile_url,
|
||||
include_geo_maps=include_geo_maps,
|
||||
)
|
||||
bundled_assets.add(archive_path.as_posix())
|
||||
if include_geo_maps:
|
||||
for filename in sorted(_GEO_MAP_ASSETS):
|
||||
map_source = _safe_runtime_source(runtime_dir, f"maps/{filename}")
|
||||
if map_source is not None:
|
||||
map_archive_path = _RUNTIME_ARCHIVE_ROOT / "maps" / filename
|
||||
_add_file(archive_files, map_archive_path, map_source)
|
||||
bundled_assets.add(map_archive_path.as_posix())
|
||||
return archive_path.as_posix()
|
||||
|
||||
source = _safe_runtime_source(runtime_dir, asset)
|
||||
if source is None:
|
||||
return match.group("url")
|
||||
archive_path = _RUNTIME_ARCHIVE_ROOT / PurePosixPath(asset)
|
||||
_add_file(archive_files, archive_path, source)
|
||||
bundled_assets.add(archive_path.as_posix())
|
||||
license_name = _LICENSE_BY_ASSET.get(asset)
|
||||
if license_name and (runtime_dir / license_name).is_file():
|
||||
licenses.add(license_name)
|
||||
return archive_path.as_posix()
|
||||
|
||||
html = runtime_url_re.sub(replace_runtime_url, html)
|
||||
|
||||
def replace_html_image(match: re.Match[str]) -> str:
|
||||
token = match.group("token")
|
||||
try:
|
||||
_thread_id, _kind, filename = verify_html_image_token(token)
|
||||
source = resolve_html_image_path(_thread_id, _kind, filename)
|
||||
except (FileNotFoundError, ValueError):
|
||||
return match.group("url")
|
||||
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()[:12]
|
||||
safe_name = Path(filename).name
|
||||
archive_path = _HTML_IMAGES_ARCHIVE_ROOT / f"{digest}-{safe_name}"
|
||||
_add_file(archive_files, archive_path, source)
|
||||
bundled_assets.add(archive_path.as_posix())
|
||||
return archive_path.as_posix()
|
||||
|
||||
html = _HTML_IMAGE_URL_RE.sub(replace_html_image, html)
|
||||
|
||||
external_url_set = {
|
||||
match.group(0).rstrip("),.;")
|
||||
for match in _EXTERNAL_URL_RE.finditer(html)
|
||||
}
|
||||
if uses_leaflet_map and str(map_tile_url).lower().startswith(("http://", "https://")):
|
||||
external_url_set.add(str(map_tile_url))
|
||||
external_urls = sorted(external_url_set)
|
||||
manifest = {
|
||||
"format": "deerflow-html-offline-bundle-v1",
|
||||
"entry": "index.html",
|
||||
"source_name": html_path.name,
|
||||
"bundled_assets": sorted(bundled_assets),
|
||||
"external_urls": external_urls,
|
||||
}
|
||||
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
||||
archive.writestr("index.html", html.encode("utf-8"))
|
||||
archive.writestr(
|
||||
"manifest.json",
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2).encode("utf-8"),
|
||||
)
|
||||
archive.writestr("README.txt", _readme(html_path.name, external_urls).encode("utf-8"))
|
||||
for archive_path, source in sorted(archive_files.items()):
|
||||
archive.write(source, archive_path)
|
||||
for archive_path, content in sorted(generated_files.items()):
|
||||
archive.writestr(archive_path, content.encode("utf-8"))
|
||||
for license_name in sorted(licenses):
|
||||
archive.write(runtime_dir / license_name, f"THIRD_PARTY_LICENSES/{license_name}")
|
||||
|
||||
return OfflineBundleResult(
|
||||
bundled_assets=tuple(sorted(bundled_assets)),
|
||||
external_urls=tuple(external_urls),
|
||||
)
|
||||
+22
-3
@@ -36,7 +36,9 @@ from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
|
||||
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
|
||||
LEASE_SECONDS = 60
|
||||
DEFAULT_LEASE_SECONDS = 5 * 60
|
||||
MIN_LEASE_SECONDS = 60
|
||||
MAX_LEASE_SECONDS = 60 * 60
|
||||
JOB_LIVE_SECONDS = 15 * 60
|
||||
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
|
||||
# A background job keeps running while Step 3 generates its artifacts, but the
|
||||
@@ -80,6 +82,15 @@ def _job_is_live(
|
||||
return updated is not None and updated >= live_after
|
||||
|
||||
|
||||
def normalize_lease_seconds(value: int | None) -> int:
|
||||
"""Clamp a foreground room lease to the supported 1–60 minute window."""
|
||||
try:
|
||||
seconds = int(value) if value is not None else DEFAULT_LEASE_SECONDS
|
||||
except (TypeError, ValueError):
|
||||
seconds = DEFAULT_LEASE_SECONDS
|
||||
return max(MIN_LEASE_SECONDS, min(MAX_LEASE_SECONDS, seconds))
|
||||
|
||||
|
||||
def _hard_deadline_reached(
|
||||
created_at: datetime | None,
|
||||
now: datetime,
|
||||
@@ -104,6 +115,7 @@ class RoundtableRoomRepository:
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
lease_seconds: int = DEFAULT_LEASE_SECONDS,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically reserve/renew a room, or reject when the pool is full."""
|
||||
@@ -119,6 +131,7 @@ class RoundtableRoomRepository:
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
lease_seconds=lease_seconds,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
@@ -133,6 +146,7 @@ class RoundtableRoomRepository:
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
lease_seconds: int = DEFAULT_LEASE_SECONDS,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically admit a pre-resolved room key.
|
||||
@@ -201,6 +215,7 @@ class RoundtableRoomRepository:
|
||||
task_id=task_id,
|
||||
holder_user_id=holder_user_id,
|
||||
now=now,
|
||||
lease_seconds=lease_seconds,
|
||||
)
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
@@ -272,6 +287,7 @@ class RoundtableRoomRepository:
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
lease_seconds: int = DEFAULT_LEASE_SECONDS,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew one live foreground lease without ever creating it.
|
||||
@@ -298,6 +314,7 @@ class RoundtableRoomRepository:
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
lease_seconds=lease_seconds,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
@@ -309,6 +326,7 @@ class RoundtableRoomRepository:
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
lease_seconds: int = DEFAULT_LEASE_SECONDS,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew a resolved persisted key without creating a missing lease."""
|
||||
@@ -354,7 +372,7 @@ class RoundtableRoomRepository:
|
||||
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
|
||||
)
|
||||
|
||||
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
row.lease_until = now + timedelta(seconds=normalize_lease_seconds(lease_seconds))
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
@@ -538,9 +556,10 @@ class RoundtableRoomRepository:
|
||||
task_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
now: datetime,
|
||||
lease_seconds: int = DEFAULT_LEASE_SECONDS,
|
||||
) -> None:
|
||||
row = await session.get(RoundtableRoomLeaseRow, room_key)
|
||||
lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
lease_until = now + timedelta(seconds=normalize_lease_seconds(lease_seconds))
|
||||
if row is None:
|
||||
session.add(
|
||||
RoundtableRoomLeaseRow(
|
||||
|
||||
@@ -314,43 +314,41 @@ def _resolve_skills_path(path: str) -> str:
|
||||
return _join_path_preserving_style(skills_host, relative)
|
||||
|
||||
|
||||
def _custom_skill_name_from_path(path: str) -> str | None:
|
||||
"""从虚拟技能路径里解析出 custom 技能名,用于"被使用"埋点。
|
||||
|
||||
只识别 ``{skills_container}/custom/{name}/...`` 形式 —— public 技能不参与
|
||||
策展,给它们埋点只会往 ``.usage.json`` 写无用条目。读技能目录下任意文件
|
||||
(SKILL.md 或 references/templates/scripts 支持文件)都算一次该技能的使用。
|
||||
解析不出技能名时返回 None。
|
||||
"""
|
||||
skills_container = _get_skills_container_path()
|
||||
if not (path == skills_container or path.startswith(f"{skills_container}/")):
|
||||
return None
|
||||
relative = path[len(skills_container) :].lstrip("/")
|
||||
parts = [segment for segment in relative.split("/") if segment]
|
||||
if len(parts) >= 2 and parts[0] == "custom":
|
||||
return parts[1]
|
||||
return None
|
||||
|
||||
|
||||
def _record_skill_read(virtual_path: str) -> None:
|
||||
"""技能文件被 ``read_file`` 读取时记一次 use —— 技能"被使用"的真实信号。
|
||||
|
||||
技能正文不进 system prompt(prompt 里只有 name/description/location),Agent
|
||||
按 Progressive Loading 用 ``read_file`` 加载 SKILL.md 才算真正用到该技能。
|
||||
curator 的年龄流转据此判断闲置,没有这个埋点会把每天都在用的技能误判为陈旧。
|
||||
best-effort:埋点失败绝不影响 ``read_file`` 本身。
|
||||
"""
|
||||
def _record_skill_paths(
|
||||
virtual_paths: list[str],
|
||||
runtime: ToolRuntime[ContextT, ThreadState] | None,
|
||||
) -> None:
|
||||
"""Record public/custom skills from real tool paths, independent of prose."""
|
||||
try:
|
||||
name = _custom_skill_name_from_path(virtual_path)
|
||||
if name is None:
|
||||
return
|
||||
from deerflow.skills.usage import bump_use
|
||||
from deerflow.sandbox.skill_view import skill_names_for_paths
|
||||
from deerflow.skills.usage import bump_use, runtime_use_event_key
|
||||
|
||||
bump_use(name)
|
||||
event_key = runtime_use_event_key(runtime)
|
||||
for name in skill_names_for_paths(
|
||||
virtual_paths,
|
||||
container_path=_get_skills_container_path(),
|
||||
):
|
||||
bump_use(name, event_key=event_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _record_skill_read(
|
||||
virtual_path: str,
|
||||
runtime: ToolRuntime[ContextT, ThreadState] | None = None,
|
||||
) -> None:
|
||||
"""Count a successful skill file read as a logical skill invocation."""
|
||||
_record_skill_paths([virtual_path], runtime)
|
||||
|
||||
|
||||
def _record_skill_command(
|
||||
command: str,
|
||||
runtime: ToolRuntime[ContextT, ThreadState] | None = None,
|
||||
) -> None:
|
||||
"""Count skills whose scripts are executed directly by ``bash``."""
|
||||
_record_skill_paths(_skill_paths_in_text(command), runtime)
|
||||
|
||||
|
||||
def _is_acp_workspace_path(path: str) -> bool:
|
||||
"""Check if a path is under the ACP workspace virtual path."""
|
||||
return path == _ACP_WORKSPACE_VIRTUAL_PATH or path.startswith(f"{_ACP_WORKSPACE_VIRTUAL_PATH}/")
|
||||
@@ -1445,6 +1443,7 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
|
||||
command: The bash command to execute. Always use absolute paths for files and directories.
|
||||
"""
|
||||
try:
|
||||
requested_command = command
|
||||
_prepare_command_skill_capability(command)
|
||||
_validate_enabled_skill_paths_in_command(command)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
@@ -1457,6 +1456,7 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
|
||||
command = replace_virtual_paths_in_command(command, thread_data)
|
||||
command = _apply_cwd_prefix(command, thread_data)
|
||||
output = sandbox.execute_command(command)
|
||||
_record_skill_command(requested_command, runtime)
|
||||
try:
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
@@ -1473,7 +1473,9 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
|
||||
max_chars = sandbox_cfg.bash_output_max_chars if sandbox_cfg else 20000
|
||||
except Exception:
|
||||
max_chars = 20000
|
||||
return _truncate_bash_output(sandbox.execute_command(command), max_chars)
|
||||
output = sandbox.execute_command(command)
|
||||
_record_skill_command(requested_command, runtime)
|
||||
return _truncate_bash_output(output, max_chars)
|
||||
except SandboxError as e:
|
||||
return f"Error: {e}"
|
||||
except PermissionError as e:
|
||||
@@ -1751,7 +1753,7 @@ def read_file_tool(
|
||||
max_result_chars=max_chars,
|
||||
hard_max_result_chars=hard_max,
|
||||
)
|
||||
_record_skill_read(requested_path)
|
||||
_record_skill_read(requested_path, runtime)
|
||||
return inspected["rendered"]
|
||||
except OSError:
|
||||
pass
|
||||
@@ -1767,7 +1769,7 @@ def read_file_tool(
|
||||
)
|
||||
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
|
||||
# 非技能路径会被 _record_skill_read 自行忽略。
|
||||
_record_skill_read(requested_path)
|
||||
_record_skill_read(requested_path, runtime)
|
||||
if not content and not truncated:
|
||||
return "(empty)"
|
||||
if truncated:
|
||||
|
||||
@@ -7,6 +7,7 @@ import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -19,6 +20,14 @@ _USAGE_FILENAME = ".usage.json"
|
||||
# 单进程跑,故进程内锁即可保证状态流转不丢。
|
||||
_usage_lock = threading.Lock()
|
||||
|
||||
# One user run can first load SKILL.md and then execute several helper scripts.
|
||||
# Those are implementation details of one logical skill invocation, so retain a
|
||||
# small process-local LRU keyed by (run, skill) and increment the public counter
|
||||
# only once. A run stays on one worker; persistence across restarts is neither
|
||||
# needed nor desirable here.
|
||||
_MAX_RECENT_USE_EVENTS = 8192
|
||||
_recent_use_events: OrderedDict[tuple[str, str], None] = OrderedDict()
|
||||
|
||||
STATE_ACTIVE = "active"
|
||||
STATE_STALE = "stale"
|
||||
STATE_ARCHIVED = "archived"
|
||||
@@ -121,8 +130,110 @@ def bump_view(name: str) -> None:
|
||||
_update_entry(name, lambda e: {"view_count": e.get("view_count", 0) + 1, "last_viewed_at": _now_iso()})
|
||||
|
||||
|
||||
def bump_use(name: str) -> None:
|
||||
_update_entry(name, lambda e: {"use_count": e.get("use_count", 0) + 1, "last_used_at": _now_iso()})
|
||||
async def _increment_use_projection(name: str, use_count: int, used_at: datetime) -> None:
|
||||
"""Keep the MySQL skill-list projection current without importing ``app``."""
|
||||
try:
|
||||
from sqlalchemy import case, update
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
return
|
||||
async with session_factory() as session:
|
||||
await session.execute(
|
||||
update(SkillRow)
|
||||
.where(SkillRow.name == name)
|
||||
.values(
|
||||
# Fire-and-forget writes may finish out of order. Never let
|
||||
# an older absolute sidecar count overwrite a newer value.
|
||||
use_count=case((SkillRow.use_count < use_count, use_count), else_=SkillRow.use_count),
|
||||
last_activity_at=case(
|
||||
(SkillRow.last_activity_at.is_(None), used_at),
|
||||
(SkillRow.last_activity_at < used_at, used_at),
|
||||
else_=SkillRow.last_activity_at,
|
||||
),
|
||||
updated_at=case(
|
||||
(SkillRow.updated_at < used_at, used_at),
|
||||
else_=SkillRow.updated_at,
|
||||
),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.debug("Failed to update skill usage projection for %r", name, exc_info=True)
|
||||
|
||||
|
||||
def _schedule_use_projection_increment(name: str, use_count: int, used_at: datetime) -> None:
|
||||
"""Best-effort bridge from synchronous sandbox tools to the DB home loop."""
|
||||
try:
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write
|
||||
|
||||
fire_and_forget_db_write(_increment_use_projection(name, use_count, used_at))
|
||||
except Exception:
|
||||
logger.debug("Failed to schedule skill usage projection for %r", name, exc_info=True)
|
||||
|
||||
|
||||
def runtime_use_event_key(runtime: Any | None) -> str | None:
|
||||
"""Return the stable user-run identifier carried by ``ToolRuntime``."""
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = context.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
config = getattr(runtime, "config", None)
|
||||
if isinstance(config, dict):
|
||||
for bucket_name in ("configurable", "metadata"):
|
||||
bucket = config.get(bucket_name)
|
||||
if not isinstance(bucket, dict):
|
||||
continue
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = bucket.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def bump_use(name: str, *, event_key: str | None = None) -> bool:
|
||||
"""Count one logical use, optionally de-duplicated by user run.
|
||||
|
||||
Returns ``True`` only when the counter changed. This makes callers and tests
|
||||
able to distinguish a new invocation from a second file/script access in
|
||||
the same run.
|
||||
"""
|
||||
normalized_name = str(name or "").strip()
|
||||
if not normalized_name:
|
||||
return False
|
||||
normalized_event = str(event_key or "").strip() or None
|
||||
dedupe_key = (normalized_event, normalized_name) if normalized_event else None
|
||||
used_at = datetime.now(UTC)
|
||||
new_count = 0
|
||||
try:
|
||||
with _usage_lock:
|
||||
if dedupe_key is not None and dedupe_key in _recent_use_events:
|
||||
_recent_use_events.move_to_end(dedupe_key)
|
||||
return False
|
||||
data = _read_all()
|
||||
entry = {**_DEFAULT_ENTRY, **data.get(normalized_name, {})}
|
||||
new_count = max(0, int(entry.get("use_count") or 0)) + 1
|
||||
entry.update({"use_count": new_count, "last_used_at": used_at.isoformat()})
|
||||
data[normalized_name] = entry
|
||||
_write_all(data)
|
||||
if dedupe_key is not None:
|
||||
_recent_use_events[dedupe_key] = None
|
||||
_recent_use_events.move_to_end(dedupe_key)
|
||||
while len(_recent_use_events) > _MAX_RECENT_USE_EVENTS:
|
||||
_recent_use_events.popitem(last=False)
|
||||
except Exception:
|
||||
logger.debug("Failed to bump usage entry for %r", normalized_name, exc_info=True)
|
||||
return False
|
||||
|
||||
_schedule_use_projection_increment(normalized_name, new_count, used_at)
|
||||
return True
|
||||
|
||||
|
||||
def bump_patch(name: str) -> None:
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
"""Filesystem-backed version history for custom skills.
|
||||
|
||||
Only the active package lives below ``skills/custom``. Historical snapshots
|
||||
are kept under the writable DeerFlow runtime directory, outside the canonical
|
||||
skills tree and therefore outside every model-facing discovery/mount path.
|
||||
|
||||
The feature is deliberately gated by ``system_settings.skill_versioning``.
|
||||
When the switch is off, callers keep the legacy write behaviour and this
|
||||
module performs no snapshot writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.skills.storage import SkillStorage, get_or_new_skill_storage
|
||||
from deerflow.skills.types import SKILL_MD_FILE, SkillCategory
|
||||
|
||||
_MANIFEST_FILE = "manifest.json"
|
||||
_SCHEMA_VERSION = 1
|
||||
_LOCKS_GUARD = threading.Lock()
|
||||
_THREAD_LOCKS: dict[str, threading.RLock] = {}
|
||||
|
||||
|
||||
def skill_versioning_enabled() -> bool:
|
||||
"""Return the deployment-wide custom-skill versioning switch."""
|
||||
|
||||
try:
|
||||
return bool(load_system_settings().skill_versioning.enabled)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _thread_lock(name: str) -> threading.RLock:
|
||||
with _LOCKS_GUARD:
|
||||
return _THREAD_LOCKS.setdefault(name, threading.RLock())
|
||||
|
||||
|
||||
def _versions_root() -> Path:
|
||||
return get_paths().base_dir / "skill-versions"
|
||||
|
||||
|
||||
def _skill_root(name: str) -> Path:
|
||||
return _versions_root() / SkillStorage.validate_skill_name(name)
|
||||
|
||||
|
||||
def _manifest_path(name: str) -> Path:
|
||||
return _skill_root(name) / _MANIFEST_FILE
|
||||
|
||||
|
||||
def _version_dir(name: str, version: int) -> Path:
|
||||
return _skill_root(name) / f"v{version:06d}"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _cross_process_lock(name: str) -> Iterator[None]:
|
||||
"""Serialize one skill's snapshot/switch operations across workers."""
|
||||
|
||||
root = _skill_root(name)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
lock_path = root / ".lock"
|
||||
with _thread_lock(name):
|
||||
with lock_path.open("a+b") as handle:
|
||||
if os.name == "nt": # pragma: no cover - exercised on Windows hosts
|
||||
import msvcrt
|
||||
|
||||
if lock_path.stat().st_size == 0:
|
||||
handle.write(b"0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else: # pragma: no cover - exercised in Linux CI/deployments
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if os.name == "nt": # pragma: no cover
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else: # pragma: no cover
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _empty_manifest(name: str) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
"skill_name": name,
|
||||
"active_version": None,
|
||||
"versions": [],
|
||||
}
|
||||
|
||||
|
||||
def _load_manifest(name: str) -> dict[str, Any]:
|
||||
path = _manifest_path(name)
|
||||
if not path.is_file():
|
||||
return _empty_manifest(name)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Skill version manifest for '{name}' is unreadable") from exc
|
||||
if not isinstance(payload, dict) or payload.get("skill_name") != name:
|
||||
raise ValueError(f"Skill version manifest for '{name}' is invalid")
|
||||
versions = payload.get("versions")
|
||||
if not isinstance(versions, list):
|
||||
raise ValueError(f"Skill version manifest for '{name}' has invalid versions")
|
||||
return payload
|
||||
|
||||
|
||||
def _save_manifest(name: str, manifest: dict[str, Any]) -> None:
|
||||
path = _manifest_path(name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(prefix=".manifest-", suffix=".tmp", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(manifest, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
os.replace(tmp_name, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp_name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _resolve_custom_dir(storage: SkillStorage, name: str) -> Path:
|
||||
for skill in storage.load_skills(enabled_only=False):
|
||||
if skill.name == name and skill.category == SkillCategory.CUSTOM:
|
||||
return skill.skill_dir.resolve()
|
||||
raise FileNotFoundError(f"Custom skill '{name}' not found")
|
||||
|
||||
|
||||
def _tree_hash(root: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if any(part in {"__pycache__", ".git"} for part in path.relative_to(root).parts):
|
||||
continue
|
||||
if path.is_symlink():
|
||||
digest.update(b"L\0")
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(os.readlink(path).encode("utf-8", errors="surrogateescape"))
|
||||
continue
|
||||
if not path.is_file():
|
||||
continue
|
||||
digest.update(b"F\0")
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _copy_package(source: Path, destination: Path) -> None:
|
||||
shutil.copytree(
|
||||
source,
|
||||
destination,
|
||||
symlinks=True,
|
||||
ignore=shutil.ignore_patterns("__pycache__", ".git"),
|
||||
)
|
||||
|
||||
|
||||
def record_custom_skill_version(
|
||||
name: str,
|
||||
*,
|
||||
storage: SkillStorage | None = None,
|
||||
action: str,
|
||||
author: str | None = None,
|
||||
summary: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Snapshot the current custom package when it differs from active.
|
||||
|
||||
Returns the active/new version metadata. No-op (and no filesystem writes)
|
||||
while the deployment switch is disabled.
|
||||
"""
|
||||
|
||||
if not skill_versioning_enabled():
|
||||
return None
|
||||
name = SkillStorage.validate_skill_name(name)
|
||||
storage = storage or get_or_new_skill_storage()
|
||||
source = _resolve_custom_dir(storage, name)
|
||||
current_hash = _tree_hash(source)
|
||||
|
||||
with _cross_process_lock(name):
|
||||
manifest = _load_manifest(name)
|
||||
active = next(
|
||||
(item for item in manifest["versions"] if int(item.get("version") or 0) == int(manifest.get("active_version") or 0)),
|
||||
None,
|
||||
)
|
||||
if active is not None and active.get("sha256") == current_hash:
|
||||
return {**active, "active": True}
|
||||
|
||||
next_version = max([int(item.get("version") or 0) for item in manifest["versions"]] or [0]) + 1
|
||||
destination = _version_dir(name, next_version)
|
||||
staging = destination.parent / f".{destination.name}.staging-{uuid4().hex}"
|
||||
_copy_package(source, staging)
|
||||
try:
|
||||
os.replace(staging, destination)
|
||||
except Exception:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
now = datetime.now(UTC).isoformat()
|
||||
entry = {
|
||||
"version": next_version,
|
||||
"sha256": current_hash,
|
||||
"created_at": now,
|
||||
"created_by": author,
|
||||
"action": str(action or "update"),
|
||||
"summary": str(summary or "").strip() or None,
|
||||
"based_on_version": manifest.get("active_version"),
|
||||
}
|
||||
manifest["versions"].append(entry)
|
||||
manifest["active_version"] = next_version
|
||||
manifest["updated_at"] = now
|
||||
_save_manifest(name, manifest)
|
||||
return {**entry, "active": True}
|
||||
|
||||
|
||||
def list_custom_skill_versions(
|
||||
name: str,
|
||||
*,
|
||||
storage: SkillStorage | None = None,
|
||||
author: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return newest-first metadata, lazily bootstrapping the current package."""
|
||||
|
||||
if not skill_versioning_enabled():
|
||||
return {"enabled": False, "skill_name": name, "active_version": None, "versions": []}
|
||||
storage = storage or get_or_new_skill_storage()
|
||||
record_custom_skill_version(
|
||||
name,
|
||||
storage=storage,
|
||||
action="baseline",
|
||||
author=author,
|
||||
summary="启用技能迭代时保存的当前版本",
|
||||
)
|
||||
with _cross_process_lock(name):
|
||||
manifest = _load_manifest(name)
|
||||
active_version = manifest.get("active_version")
|
||||
versions = [{**item, "active": int(item.get("version") or 0) == int(active_version or 0)} for item in reversed(manifest["versions"])]
|
||||
return {
|
||||
"enabled": True,
|
||||
"skill_name": name,
|
||||
"active_version": active_version,
|
||||
"versions": versions,
|
||||
}
|
||||
|
||||
|
||||
def activate_custom_skill_version(
|
||||
name: str,
|
||||
version: int,
|
||||
*,
|
||||
storage: SkillStorage | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Materialize one historical snapshot as the sole model-visible package."""
|
||||
|
||||
if not skill_versioning_enabled():
|
||||
raise RuntimeError("技能迭代功能未开启")
|
||||
name = SkillStorage.validate_skill_name(name)
|
||||
storage = storage or get_or_new_skill_storage()
|
||||
target = _resolve_custom_dir(storage, name)
|
||||
|
||||
with _cross_process_lock(name):
|
||||
manifest = _load_manifest(name)
|
||||
entry = next(
|
||||
(item for item in manifest["versions"] if int(item.get("version") or 0) == int(version)),
|
||||
None,
|
||||
)
|
||||
if entry is None:
|
||||
raise FileNotFoundError(f"Skill '{name}' version {version} not found")
|
||||
source = _version_dir(name, int(version))
|
||||
if not source.is_dir() or _tree_hash(source) != entry.get("sha256"):
|
||||
raise ValueError(f"Skill '{name}' version {version} snapshot is missing or corrupted")
|
||||
storage.validate_skill_markdown_content(
|
||||
name,
|
||||
(source / SKILL_MD_FILE).read_text(encoding="utf-8"),
|
||||
)
|
||||
if int(manifest.get("active_version") or 0) == int(version) and _tree_hash(target) == entry.get("sha256"):
|
||||
return {**entry, "active": True}
|
||||
|
||||
parent = target.parent
|
||||
staging = parent / f".{name}.version-stage-{uuid4().hex}"
|
||||
backup = parent / f".{name}.version-backup-{uuid4().hex}"
|
||||
_copy_package(source, staging)
|
||||
moved_old = False
|
||||
try:
|
||||
os.replace(target, backup)
|
||||
moved_old = True
|
||||
os.replace(staging, target)
|
||||
manifest["active_version"] = int(version)
|
||||
manifest["updated_at"] = datetime.now(UTC).isoformat()
|
||||
_save_manifest(name, manifest)
|
||||
except Exception:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
if moved_old:
|
||||
shutil.rmtree(target, ignore_errors=True)
|
||||
os.replace(backup, target)
|
||||
raise
|
||||
finally:
|
||||
if backup.exists() and target.exists():
|
||||
shutil.rmtree(backup, ignore_errors=True)
|
||||
return {**entry, "active": True}
|
||||
|
||||
|
||||
def delete_custom_skill_versions(name: str) -> None:
|
||||
"""Remove stored snapshots after the custom skill itself is deleted."""
|
||||
|
||||
name = SkillStorage.validate_skill_name(name)
|
||||
root = _skill_root(name)
|
||||
if root.exists():
|
||||
shutil.rmtree(root)
|
||||
|
||||
|
||||
def reset_custom_skill_versions(
|
||||
name: str,
|
||||
*,
|
||||
storage: SkillStorage | None = None,
|
||||
author: str | None = None,
|
||||
summary: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Discard the previous chain and make the current package the new v1.
|
||||
|
||||
This is the explicit "direct overwrite" semantic offered beside
|
||||
"iterate as a new version" for same-name uploads.
|
||||
"""
|
||||
|
||||
if not skill_versioning_enabled():
|
||||
return None
|
||||
name = SkillStorage.validate_skill_name(name)
|
||||
root = _skill_root(name)
|
||||
with _cross_process_lock(name):
|
||||
for child in list(root.iterdir()):
|
||||
if child.name == ".lock":
|
||||
continue
|
||||
if child.is_dir():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
child.unlink(missing_ok=True)
|
||||
return record_custom_skill_version(
|
||||
name,
|
||||
storage=storage,
|
||||
action="overwrite",
|
||||
author=author,
|
||||
summary=summary or "直接覆盖后重置版本链",
|
||||
)
|
||||
|
||||
|
||||
def reconcile_custom_skill_versions(*, storage: SkillStorage | None = None, author: str | None = None) -> int:
|
||||
"""Capture each current custom skill after the master switch is enabled."""
|
||||
|
||||
if not skill_versioning_enabled():
|
||||
return 0
|
||||
storage = storage or get_or_new_skill_storage()
|
||||
captured = 0
|
||||
for skill in storage.load_skills(enabled_only=False):
|
||||
if skill.category != SkillCategory.CUSTOM:
|
||||
continue
|
||||
before = _load_manifest(skill.name).get("active_version") if _manifest_path(skill.name).exists() else None
|
||||
result = record_custom_skill_version(
|
||||
skill.name,
|
||||
storage=storage,
|
||||
action="feature_enabled",
|
||||
author=author,
|
||||
summary="技能迭代开关开启后捕获当前内容",
|
||||
)
|
||||
if result is not None and result.get("version") != before:
|
||||
captured += 1
|
||||
return captured
|
||||
|
||||
|
||||
__all__ = [
|
||||
"activate_custom_skill_version",
|
||||
"delete_custom_skill_versions",
|
||||
"list_custom_skill_versions",
|
||||
"record_custom_skill_version",
|
||||
"reconcile_custom_skill_versions",
|
||||
"reset_custom_skill_versions",
|
||||
"skill_versioning_enabled",
|
||||
]
|
||||
+10
-1
@@ -273,7 +273,16 @@ def skill_view_tool(name: str, runtime: ToolRuntime) -> str:
|
||||
Args:
|
||||
name: The skill name (hyphen-case, e.g. 'my-skill').
|
||||
"""
|
||||
return skill_view_impl(name, _resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
|
||||
result = skill_view_impl(name, _resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
|
||||
head = result.lstrip()[:200].lower()
|
||||
if not ((head.startswith("skill '") and "not found" in head) or head.startswith("error reading skill")):
|
||||
try:
|
||||
from deerflow.skills.usage import bump_use, runtime_use_event_key
|
||||
|
||||
bump_use(name, event_key=runtime_use_event_key(runtime))
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_skills_container_base_path() -> str:
|
||||
|
||||
@@ -161,6 +161,26 @@ async def _publish_managed_skill(name: str, *, removed: bool = False) -> None:
|
||||
await _sync_managed_skill_view(name)
|
||||
|
||||
|
||||
async def _record_managed_skill_version(
|
||||
storage: SkillStorage,
|
||||
name: str,
|
||||
*,
|
||||
action: str,
|
||||
author: str,
|
||||
summary: str,
|
||||
) -> None:
|
||||
from deerflow.skills.versioning import record_custom_skill_version
|
||||
|
||||
await _to_thread(
|
||||
record_custom_skill_version,
|
||||
name,
|
||||
storage=storage,
|
||||
action=action,
|
||||
author=author,
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
|
||||
async def _bump_patch_best_effort(name: str) -> None:
|
||||
"""记一次技能维护活动(patch / edit)。
|
||||
|
||||
@@ -363,6 +383,14 @@ async def _skill_manage_impl(
|
||||
# async stack — contextvars don't propagate into ``to_thread``.
|
||||
if skill_store is not None:
|
||||
await _ensure_skill_db_registered(skill_store, name, user_id)
|
||||
skill_storage.invalidate_skills_cache()
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="agent_create",
|
||||
author=user_id,
|
||||
summary="大模型创建技能",
|
||||
)
|
||||
await _publish_managed_skill(name)
|
||||
return f"Created custom skill '{name}'."
|
||||
|
||||
@@ -378,6 +406,13 @@ async def _skill_manage_impl(
|
||||
scan = await _scan_or_raise(content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
|
||||
skill_file = skill_storage.get_custom_skill_file(name)
|
||||
prev_content = await _to_thread(skill_file.read_text, encoding="utf-8")
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="baseline",
|
||||
author=user_id,
|
||||
summary="大模型编辑前的活动版本",
|
||||
)
|
||||
await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, content)
|
||||
await _to_thread(
|
||||
skill_storage.append_history,
|
||||
@@ -385,6 +420,13 @@ async def _skill_manage_impl(
|
||||
_history_record(action="edit", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _bump_patch_best_effort(name)
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="agent_edit",
|
||||
author=user_id,
|
||||
summary="大模型编辑 SKILL.md",
|
||||
)
|
||||
await _publish_managed_skill(name)
|
||||
return f"Updated custom skill '{name}'."
|
||||
|
||||
@@ -406,6 +448,13 @@ async def _skill_manage_impl(
|
||||
new_content = prev_content.replace(find, replace, replacement_count)
|
||||
await _to_thread(skill_storage.validate_skill_markdown_content, name, new_content)
|
||||
scan = await _scan_or_raise(new_content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="baseline",
|
||||
author=user_id,
|
||||
summary="大模型修补前的活动版本",
|
||||
)
|
||||
await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, new_content)
|
||||
await _to_thread(
|
||||
skill_storage.append_history,
|
||||
@@ -413,6 +462,13 @@ async def _skill_manage_impl(
|
||||
_history_record(action="patch", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=new_content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _bump_patch_best_effort(name)
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="agent_patch",
|
||||
author=user_id,
|
||||
summary="大模型修补 SKILL.md",
|
||||
)
|
||||
await _publish_managed_skill(name)
|
||||
return f"Patched custom skill '{name}' ({replacement_count} replacement(s) applied, {occurrences} match(es) found)."
|
||||
|
||||
@@ -453,6 +509,12 @@ async def _skill_manage_impl(
|
||||
await skill_store.delete(name, user_id)
|
||||
except Exception:
|
||||
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
|
||||
try:
|
||||
from deerflow.skills.versioning import delete_custom_skill_versions
|
||||
|
||||
await _to_thread(delete_custom_skill_versions, name)
|
||||
except Exception:
|
||||
logger.debug("Failed to delete version history for %r", name, exc_info=True)
|
||||
agent_store = _default_agent_store()
|
||||
if agent_store is not None:
|
||||
try:
|
||||
@@ -482,12 +544,26 @@ async def _skill_manage_impl(
|
||||
prev_content = await _to_thread(target.read_text, encoding="utf-8") if exists else None
|
||||
executable = "scripts/" in path or path.startswith("scripts/")
|
||||
scan = await _scan_or_raise(content, executable=executable, location=f"{name}/{path}")
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="baseline",
|
||||
author=user_id,
|
||||
summary="大模型写文件前的活动版本",
|
||||
)
|
||||
await _to_thread(skill_storage.write_custom_skill, name, path, content)
|
||||
await _to_thread(
|
||||
skill_storage.append_history,
|
||||
name,
|
||||
_history_record(action="write_file", file_path=path, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="agent_write_file",
|
||||
author=user_id,
|
||||
summary=f"大模型写入文件 {path}",
|
||||
)
|
||||
await _sync_managed_skill_view(name)
|
||||
return f"Wrote '{path}' for custom skill '{name}'."
|
||||
|
||||
@@ -502,12 +578,26 @@ async def _skill_manage_impl(
|
||||
if not await _to_thread(target.exists):
|
||||
raise FileNotFoundError(f"Supporting file '{path}' not found for skill '{name}'.")
|
||||
prev_content = await _to_thread(target.read_text, encoding="utf-8")
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="baseline",
|
||||
author=user_id,
|
||||
summary="大模型删文件前的活动版本",
|
||||
)
|
||||
await _to_thread(target.unlink)
|
||||
await _to_thread(
|
||||
skill_storage.append_history,
|
||||
name,
|
||||
_history_record(action="remove_file", file_path=path, prev_content=prev_content, new_content=None, thread_id=thread_id, scanner={"decision": "allow", "reason": "Deletion requested."}),
|
||||
)
|
||||
await _record_managed_skill_version(
|
||||
skill_storage,
|
||||
name,
|
||||
action="agent_remove_file",
|
||||
author=user_id,
|
||||
summary=f"大模型删除文件 {path}",
|
||||
)
|
||||
await _sync_managed_skill_view(name)
|
||||
return f"Removed '{path}' from custom skill '{name}'."
|
||||
|
||||
|
||||
+10
-8
@@ -5,26 +5,28 @@ import pytest
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
_SKILL_VIEW_KEYS = ("skill_view_reuse_enabled", "skill_view_cache_enabled")
|
||||
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled", "tool_metrics_enabled")
|
||||
_DEFAULT_ON_KEYS = (*_SKILL_VIEW_KEYS, "tool_metrics_enabled", "skill_metrics_enabled")
|
||||
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled")
|
||||
|
||||
|
||||
def test_missing_runtime_performance_enables_skill_view_only():
|
||||
def test_missing_runtime_performance_enables_skill_view_and_skill_metrics():
|
||||
config = AppConfig(sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"})
|
||||
assert config.runtime_performance.model_dump() == {
|
||||
"skill_view_reuse_enabled": True,
|
||||
"skill_view_cache_enabled": True,
|
||||
"tool_metrics_enabled": False,
|
||||
"tool_metrics_enabled": True,
|
||||
"skill_metrics_enabled": True,
|
||||
"model_timing_nonblocking_enabled": False,
|
||||
"asyncio_thread_pool_enabled": False,
|
||||
"asyncio_thread_pool_size": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [*_SKILL_VIEW_KEYS, *_OPT_IN_KEYS])
|
||||
@pytest.mark.parametrize("key", [*_DEFAULT_ON_KEYS, *_OPT_IN_KEYS])
|
||||
def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
|
||||
# Skill-view switches default on, so independence is checked by turning
|
||||
# one of them off. The opt-in switches default off and are turned on.
|
||||
enabled = key not in _SKILL_VIEW_KEYS
|
||||
# Skill-view and audit metrics default on, so independence is checked by
|
||||
# turning one off. Opt-in switches default off and are turned on.
|
||||
enabled = key not in _DEFAULT_ON_KEYS
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
f"sandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\nruntime_performance:\n {key}: {'true' if enabled else 'false'}\n",
|
||||
@@ -32,7 +34,7 @@ def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
|
||||
)
|
||||
flags = AppConfig.from_file(str(config_path)).runtime_performance.model_dump()
|
||||
assert flags[key] is enabled
|
||||
for other in _SKILL_VIEW_KEYS:
|
||||
for other in _DEFAULT_ON_KEYS:
|
||||
if other != key:
|
||||
assert flags[other] is True
|
||||
for other in _OPT_IN_KEYS:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Regression coverage for model-independent skill usage attribution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.sandbox import tools
|
||||
from deerflow.skills import usage
|
||||
from deerflow.tools.builtins import skill_tools
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_recent_use_events():
|
||||
usage._recent_use_events.clear()
|
||||
yield
|
||||
usage._recent_use_events.clear()
|
||||
|
||||
|
||||
def test_bump_use_counts_a_skill_once_per_run_and_syncs_the_catalog(tmp_path: Path, monkeypatch):
|
||||
usage_path = tmp_path / ".usage.json"
|
||||
projected: list[tuple[str, int, object]] = []
|
||||
monkeypatch.setattr(usage, "_usage_path", lambda: usage_path)
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"_schedule_use_projection_increment",
|
||||
lambda name, use_count, used_at: projected.append((name, use_count, used_at)),
|
||||
)
|
||||
|
||||
assert usage.bump_use("page-generator", event_key="run-1") is True
|
||||
assert usage.bump_use("page-generator", event_key="run-1") is False
|
||||
assert usage.bump_use("page-generator", event_key="run-2") is True
|
||||
|
||||
entry = usage.get_entry("page-generator")
|
||||
assert entry["use_count"] == 2
|
||||
assert entry["last_used_at"]
|
||||
assert [(name, count) for name, count, _used_at in projected] == [
|
||||
("page-generator", 1),
|
||||
("page-generator", 2),
|
||||
]
|
||||
|
||||
|
||||
def test_skill_file_and_script_paths_record_public_and_custom_skills(monkeypatch):
|
||||
calls: list[tuple[str, str | None]] = []
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"bump_use",
|
||||
lambda name, *, event_key=None: calls.append((name, event_key)),
|
||||
)
|
||||
runtime = SimpleNamespace(context={"run_id": "run-weak-model"}, config={})
|
||||
|
||||
tools._record_skill_read("/mnt/skills/public/report-maker/SKILL.md", runtime)
|
||||
tools._record_skill_command(
|
||||
"python /mnt/skills/custom/data-cleaner/scripts/run.py --fast",
|
||||
runtime,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("report-maker", "run-weak-model"),
|
||||
("data-cleaner", "run-weak-model"),
|
||||
]
|
||||
|
||||
|
||||
def test_skill_view_counts_as_use_without_model_narration(monkeypatch):
|
||||
calls: list[tuple[str, str | None]] = []
|
||||
runtime = SimpleNamespace(context={"run_id": "run-skill-view"}, config={})
|
||||
monkeypatch.setattr(skill_tools, "skill_view_impl", lambda *_args, **_kwargs: "skill instructions")
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"bump_use",
|
||||
lambda name, *, event_key=None: calls.append((name, event_key)),
|
||||
)
|
||||
|
||||
result = skill_tools.skill_view_tool.func(name="report-maker", runtime=runtime)
|
||||
|
||||
assert result == "skill instructions"
|
||||
assert calls == [("report-maker", "run-skill-view")]
|
||||
|
||||
|
||||
def test_failed_skill_view_does_not_count_as_use(monkeypatch):
|
||||
calls: list[str] = []
|
||||
runtime = SimpleNamespace(context={"run_id": "run-missing"}, config={})
|
||||
monkeypatch.setattr(
|
||||
skill_tools,
|
||||
"skill_view_impl",
|
||||
lambda *_args, **_kwargs: "Skill 'missing' not found.",
|
||||
)
|
||||
monkeypatch.setattr(usage, "bump_use", lambda name, **_kwargs: calls.append(name))
|
||||
|
||||
result = skill_tools.skill_view_tool.func(name="missing", runtime=runtime)
|
||||
|
||||
assert result == "Skill 'missing' not found."
|
||||
assert calls == []
|
||||
+25
-1
@@ -288,7 +288,7 @@ def _tool_sandbox(tmp_path: Path, view_root: Path, monkeypatch: pytest.MonkeyPat
|
||||
monkeypatch.setattr(tools, "is_local_sandbox", lambda _runtime: True)
|
||||
monkeypatch.setattr(tools, "is_host_bash_allowed", lambda: True)
|
||||
monkeypatch.setattr(tools, "get_thread_data", lambda _runtime: {"workspace_path": str(tmp_path / "workspace"), "uploads_path": str(tmp_path / "uploads"), "outputs_path": str(tmp_path / "outputs")})
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda _path: None)
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda *_args: None)
|
||||
monkeypatch.setattr(tools, "_get_skills_container_path", lambda: "/mnt/skills")
|
||||
(tmp_path / "workspace").mkdir(exist_ok=True)
|
||||
assert view_root.is_dir()
|
||||
@@ -326,6 +326,30 @@ def test_bash_runs_the_updated_skill_script(cache_setup, monkeypatch, tmp_path):
|
||||
assert (view.root / "public" / "online" / "scripts" / "run.py").read_text(encoding="utf-8") == "print('two')\n"
|
||||
|
||||
|
||||
def test_bash_records_a_direct_skill_script_invocation(cache_setup, monkeypatch, tmp_path):
|
||||
root, _config, storage = cache_setup
|
||||
script = root / "public" / "online" / "scripts"
|
||||
script.mkdir()
|
||||
(script / "run.py").write_text("print('used')\n", encoding="utf-8")
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
recorded: list[tuple[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
tools,
|
||||
"_record_skill_command",
|
||||
lambda command, runtime: recorded.append((command, runtime)),
|
||||
)
|
||||
|
||||
result = tools.bash_tool.func(
|
||||
runtime=None,
|
||||
description="test",
|
||||
command="python /mnt/skills/public/online/scripts/run.py",
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert recorded == [("python /mnt/skills/public/online/scripts/run.py", None)]
|
||||
|
||||
|
||||
def test_bash_rejects_an_offline_skill(cache_setup, monkeypatch, tmp_path):
|
||||
root, config, storage = cache_setup
|
||||
_skill(root, "offline")
|
||||
|
||||
+18
@@ -50,3 +50,21 @@ def test_canonical_write_file_is_still_recorded(monkeypatch) -> None:
|
||||
middleware.wrap_tool_call(_request("write_file"), lambda _: _result("write_file"))
|
||||
|
||||
flush.assert_called_once()
|
||||
|
||||
|
||||
def test_call_metrics_switch_skips_new_metric_rows(monkeypatch) -> None:
|
||||
middleware = ToolMetricsMiddleware()
|
||||
flush = Mock()
|
||||
monkeypatch.setattr(middleware, "_flush", flush)
|
||||
monkeypatch.setattr(
|
||||
"deerflow.agents.middlewares.tool_metrics_middleware.get_call_metrics_settings",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
|
||||
result = middleware.wrap_tool_call(
|
||||
_request("write_file"),
|
||||
lambda _: _result("write_file"),
|
||||
)
|
||||
|
||||
assert result.name == "write_file"
|
||||
flush.assert_not_called()
|
||||
@@ -20,8 +20,10 @@ from deerflow.config.app_config import AppConfig
|
||||
BUILDERS = [build_lead_runtime_middlewares, build_subagent_runtime_middlewares]
|
||||
|
||||
|
||||
def _config(enabled=None):
|
||||
def _config(enabled=None, *, skill_enabled=None):
|
||||
options = {} if enabled is None else {"tool_metrics_enabled": enabled}
|
||||
if skill_enabled is not None:
|
||||
options["skill_metrics_enabled"] = skill_enabled
|
||||
return AppConfig(
|
||||
sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
runtime_performance=options,
|
||||
@@ -29,18 +31,29 @@ def _config(enabled=None):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
@pytest.mark.parametrize("enabled", [None, False])
|
||||
def test_paused_runtime_never_constructs_a_recorder(builder, enabled, monkeypatch):
|
||||
def test_explicitly_paused_runtime_never_constructs_a_recorder(builder, monkeypatch):
|
||||
constructor = Mock(side_effect=AssertionError("paused runtime constructed a DB recorder"))
|
||||
monkeypatch.setattr(metrics, "ToolMetricsMiddleware", constructor)
|
||||
|
||||
middlewares = builder(app_config=_config(enabled))
|
||||
middlewares = builder(app_config=_config(False, skill_enabled=False))
|
||||
|
||||
constructor.assert_not_called()
|
||||
assert any(isinstance(m, ToolErrorHandlingMiddleware) for m in middlewares)
|
||||
assert any(isinstance(m, LlmMetricsMiddleware) for m in middlewares)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
def test_default_runtime_keeps_a_full_tool_and_skill_recorder(builder):
|
||||
recorders = [
|
||||
middleware
|
||||
for middleware in builder(app_config=_config())
|
||||
if isinstance(middleware, metrics.ToolMetricsMiddleware)
|
||||
]
|
||||
|
||||
assert len(recorders) == 1
|
||||
assert recorders[0]._skill_only is False
|
||||
|
||||
|
||||
def _tool_wrappers(builder, enabled):
|
||||
# Exercise the actual error/audit wrappers selected by both runtime builders.
|
||||
return [m for m in builder(app_config=_config(enabled)) if isinstance(m, (ToolErrorHandlingMiddleware, metrics.ToolMetricsMiddleware))]
|
||||
@@ -77,6 +90,30 @@ def test_sync_tool_execution_and_error_conversion_survive_pause(builder, enabled
|
||||
assert flushed.call_args.args[0].status == result.status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
def test_skill_only_mode_records_a_path_based_skill_call(builder, monkeypatch):
|
||||
flushed = Mock()
|
||||
monkeypatch.setattr(metrics.ToolMetricsMiddleware, "_flush", lambda _, record: flushed(record))
|
||||
request = SimpleNamespace(
|
||||
tool_call={
|
||||
"id": "call-skill",
|
||||
"name": "bash",
|
||||
"args": {"command": "python /mnt/skills/public/report-maker/scripts/run.py"},
|
||||
}
|
||||
)
|
||||
expected = ToolMessage(content="ok", tool_call_id="call-skill", name="bash")
|
||||
|
||||
def handler(_request):
|
||||
return expected
|
||||
|
||||
for middleware in reversed(_tool_wrappers(builder, False)):
|
||||
handler = partial(middleware.wrap_tool_call, handler=handler)
|
||||
|
||||
assert handler(request) is expected
|
||||
assert flushed.call_count == 1
|
||||
assert flushed.call_args.args[0].skill_name == "report-maker"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
|
||||
Reference in new issue
Block a user