sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -1,6 +1,6 @@
|
||||
from .tools import get_available_tools
|
||||
from .tools import get_available_tools, get_available_tools_async
|
||||
|
||||
__all__ = ["get_available_tools", "skill_manage_tool"]
|
||||
__all__ = ["get_available_tools", "get_available_tools_async", "skill_manage_tool"]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
|
||||
+30
-3
@@ -31,6 +31,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Annotated, Any
|
||||
|
||||
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
|
||||
@@ -296,6 +297,21 @@ def strip_solution_numeric_citations(markdown: str) -> str:
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _report_requires_unnumbered_sources(runtime_context: Mapping[str, Any]) -> bool:
|
||||
"""Return whether this run's report must avoid numbered source markers."""
|
||||
|
||||
project_context = runtime_context.get("analysis_project")
|
||||
project_mode = (
|
||||
str(project_context.get("mode") or "").strip().lower()
|
||||
if isinstance(project_context, Mapping)
|
||||
else ""
|
||||
)
|
||||
retrieval_mode = str(
|
||||
runtime_context.get("knowledge_retrieval_mode") or "classic"
|
||||
).strip().lower()
|
||||
return project_mode == "solution" or retrieval_mode == "smart_reasoning"
|
||||
|
||||
|
||||
_WORKBENCH_CONFIG_KEYS = frozenset(
|
||||
{
|
||||
"custom_outline",
|
||||
@@ -325,6 +341,7 @@ def build_writing_research_config(
|
||||
project_mode: str | None = None,
|
||||
project_report_config: dict | None = None,
|
||||
workbench_report_config: dict | None = None,
|
||||
unnumbered_sources: bool = False,
|
||||
) -> DeepResearchConfig:
|
||||
"""组装聊天写作模式的研究配置:basic + 通用分析结构 + adaptive。
|
||||
|
||||
@@ -336,6 +353,7 @@ def build_writing_research_config(
|
||||
工作台确认卡传入 ``workbench_report_config`` 时沿用卡片上的大纲/结构/配图
|
||||
等选项,但 **mode 仍强制 basic**:那是普通问答写作模式在脆弱服务器上能
|
||||
写完的同一条 runner,避免再走 deep/multi_agent 的 job 路径。
|
||||
``unnumbered_sources`` 只改变引用写法,不切换方案模式或固定大纲。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
if focus and focus.strip():
|
||||
@@ -352,6 +370,8 @@ def build_writing_research_config(
|
||||
if extra_instruction:
|
||||
parts.append(extra_instruction)
|
||||
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
|
||||
if fixed_solution or unnumbered_sources:
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
return DeepResearchConfig(
|
||||
mode="basic",
|
||||
collection_mode="legacy",
|
||||
@@ -380,6 +400,8 @@ def build_writing_research_config(
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
else:
|
||||
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
|
||||
if unnumbered_sources:
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
return DeepResearchConfig(
|
||||
mode="basic",
|
||||
collection_mode="legacy",
|
||||
@@ -632,7 +654,7 @@ async def deep_research_report_tool(
|
||||
|
||||
The tool harvests the materials already produced in this conversation (search /
|
||||
knowledge / skill / file tool results) and writes a full research report
|
||||
(general-analysis structure, numbered reference section) to
|
||||
(general-analysis structure, with source provenance) to
|
||||
/mnt/user-data/outputs/report.md in this thread. The file is presented
|
||||
automatically; do NOT write the report body yourself with write_file.
|
||||
|
||||
@@ -691,7 +713,11 @@ async def deep_research_report_tool(
|
||||
workbench_report_config = _coerce_workbench_report_config(runtime_context.get("writing_report_config"))
|
||||
workbench_session_id = str(runtime_context.get("workbench_session_id") or "").strip()
|
||||
force_no_materials = bool(runtime_context.get("force_no_materials"))
|
||||
harvest_options = {"include_rag_sources": True} if project_context.get("mode") == "solution" else {}
|
||||
harvest_options = (
|
||||
{"include_rag_sources": True}
|
||||
if project_context.get("mode") == "solution" or workbench_session_id
|
||||
else {}
|
||||
)
|
||||
if force_no_materials:
|
||||
harvested, dropped_count, harvested_count = [], 0, 0
|
||||
materials: list[ResearchMaterial] = []
|
||||
@@ -742,6 +768,7 @@ async def deep_research_report_tool(
|
||||
project_mode=project_context.get("mode"),
|
||||
project_report_config=project_context.get("report_config"),
|
||||
workbench_report_config=workbench_report_config or None,
|
||||
unnumbered_sources=_report_requires_unnumbered_sources(runtime_context),
|
||||
)
|
||||
if workbench_session_id:
|
||||
await _persist_workbench_session(
|
||||
@@ -781,7 +808,7 @@ async def deep_research_report_tool(
|
||||
error_message: str | None = None
|
||||
try:
|
||||
generated_result = await DeepResearchEngine().run(request, adapters=bundle)
|
||||
if project_context.get("mode") == "solution":
|
||||
if _report_requires_unnumbered_sources(runtime_context):
|
||||
cleaned_report = strip_solution_numeric_citations(generated_result.report_markdown)
|
||||
if cleaned_report != generated_result.report_markdown:
|
||||
await bundle.artifacts.write_text("report.md", cleaned_report)
|
||||
|
||||
+1958
File diff suppressed because it is too large.
Load diff
+75
-2
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
@@ -98,6 +99,42 @@ def _sanitize_conversation_deposit_value(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _stable_citation_id(item: Mapping[str, Any]) -> str:
|
||||
"""Build a stable cross-call identity without exposing provider IDs."""
|
||||
|
||||
knowledge_base_id = str(
|
||||
item.get("knowledge_base_id") or item.get("base_id") or "unknown"
|
||||
)
|
||||
identity = next(
|
||||
(
|
||||
str(item.get(key) or "").strip()
|
||||
for key in (
|
||||
"chunk_id",
|
||||
"page_ref",
|
||||
"wiki_slug",
|
||||
"slug",
|
||||
"knowledge_id",
|
||||
"id",
|
||||
"title",
|
||||
)
|
||||
if str(item.get(key) or "").strip()
|
||||
),
|
||||
"result",
|
||||
)
|
||||
digest = hashlib.sha256(f"{knowledge_base_id}\0{identity}".encode()).hexdigest()[:20]
|
||||
return f"llmwiki:{digest}"
|
||||
|
||||
|
||||
def _result_score(item: Mapping[str, Any]) -> float:
|
||||
for key in ("relevance_score", "rerank_score", "score", "similarity"):
|
||||
try:
|
||||
value = float(item.get(key))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return value
|
||||
return 0.0
|
||||
|
||||
|
||||
def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
"""Read the conversation selection; ``None`` means legacy/no selection key."""
|
||||
|
||||
@@ -120,7 +157,7 @@ def selected_assistant_knowledge_base_ids(context: Mapping[str, Any] | None) ->
|
||||
|
||||
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
agent_id = str((context or {}).get("agent_id") or "").strip()
|
||||
if not agent_id or agent_id in {"default", "lead_agent"}:
|
||||
if not agent_id or agent_id in {"default", "lead_agent", "deep-research-planner", "deep-research-collector"}:
|
||||
return None
|
||||
try:
|
||||
config = load_agent_config(agent_id)
|
||||
@@ -133,6 +170,7 @@ def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
async def llmwiki_search_tool(
|
||||
query: str,
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
knowledge_base_id: str = "",
|
||||
) -> str:
|
||||
"""Search the LLMWiki knowledge selected for this conversation.
|
||||
|
||||
@@ -142,6 +180,9 @@ async def llmwiki_search_tool(
|
||||
|
||||
Args:
|
||||
query: Focused natural-language retrieval query.
|
||||
knowledge_base_id: Optional selected DeerFlow knowledge-base ID. Use one
|
||||
ID per call when several libraries are selected so each library is
|
||||
searched and diagnosed independently.
|
||||
"""
|
||||
|
||||
started_at = perf_counter()
|
||||
@@ -153,9 +194,36 @@ async def llmwiki_search_tool(
|
||||
context = runtime.context if runtime.context else {}
|
||||
selected_ids = selected_knowledge_base_ids(context)
|
||||
assistant_selected_ids = selected_assistant_knowledge_base_ids(context)
|
||||
allowed_ids = _agent_allowed_ids(context)
|
||||
selected_ids = selected_ids or []
|
||||
assistant_selected_ids = assistant_selected_ids or []
|
||||
retrieval_mode = str(context.get("knowledge_retrieval_mode") or "classic").strip().lower()
|
||||
if retrieval_mode != "classic":
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=selected_ids,
|
||||
assistant_selection=assistant_selected_ids,
|
||||
zero_result_reason="CLASSIC_MODE_REQUIRED",
|
||||
error="llmwiki_search is available only in classic knowledge retrieval mode.",
|
||||
)
|
||||
allowed_ids = _agent_allowed_ids(context)
|
||||
requested_knowledge_base_id = str(knowledge_base_id or "").strip()
|
||||
if requested_knowledge_base_id:
|
||||
if requested_knowledge_base_id in selected_ids:
|
||||
selected_ids = [requested_knowledge_base_id]
|
||||
assistant_selected_ids = []
|
||||
elif requested_knowledge_base_id in assistant_selected_ids:
|
||||
selected_ids = []
|
||||
assistant_selected_ids = [requested_knowledge_base_id]
|
||||
else:
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=selected_ids,
|
||||
assistant_selection=assistant_selected_ids,
|
||||
zero_result_reason="KNOWLEDGE_SCOPE_DENIED",
|
||||
error="knowledge_base_id is not selected for this conversation.",
|
||||
)
|
||||
if not selected_ids and not assistant_selected_ids:
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
@@ -323,6 +391,11 @@ async def llmwiki_search_tool(
|
||||
else:
|
||||
mode = str(payload.get("mode") or "empty")
|
||||
|
||||
results.sort(key=_result_score, reverse=True)
|
||||
for rank, item in enumerate(results, start=1):
|
||||
item.setdefault("citation_id", _stable_citation_id(item))
|
||||
item.setdefault("relevance_rank", rank)
|
||||
|
||||
if results:
|
||||
zero_result_reason = None
|
||||
elif fallback_reason:
|
||||
|
||||
+123
-16
@@ -62,7 +62,40 @@ def _required_visualizations(features: Iterable[str], material_brief: str = "")
|
||||
return required
|
||||
|
||||
|
||||
def _active_html_map_engine() -> str:
|
||||
from deerflow.html_runtime.map_engine import html_map_engine
|
||||
|
||||
return html_map_engine()
|
||||
|
||||
|
||||
_RUNTIME_BASE_PLACEHOLDER = "__DF_HTML_RUNTIME_BASE__"
|
||||
|
||||
|
||||
def _html_runtime_base_url() -> str:
|
||||
"""Browser-reachable runtime base saved in the settings dialog."""
|
||||
from deerflow.html_runtime.addresses import html_runtime_base_url
|
||||
|
||||
return html_runtime_base_url()
|
||||
|
||||
|
||||
def _visualization_contract(required: set[str]) -> dict[str, object]:
|
||||
leaflet = _active_html_map_engine() == "leaflet"
|
||||
map_contract: dict[str, object] = (
|
||||
{
|
||||
"renderer": "Leaflet 地图网页",
|
||||
"required_assets": ["leaflet.css", "leaflet.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-map\"", "data-df-map"],
|
||||
"forbidden": ["data-df-geo", "df-geo-map", "manual L.map", "tile URL in HTML"],
|
||||
}
|
||||
if leaflet
|
||||
else {
|
||||
"renderer": "ECharts GeoJSON(Canvas)",
|
||||
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
|
||||
"supported_maps": sorted(_GEO_MAP_CATALOG),
|
||||
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
|
||||
}
|
||||
)
|
||||
contracts: dict[str, dict[str, object]] = {
|
||||
"flowchart": {
|
||||
"renderer": "AntV G6",
|
||||
@@ -75,13 +108,7 @@ def _visualization_contract(required: set[str]) -> dict[str, object]:
|
||||
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-chart\"", "data-df-chart"],
|
||||
},
|
||||
"map": {
|
||||
"renderer": "ECharts GeoJSON(Canvas)",
|
||||
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
|
||||
"supported_maps": sorted(_GEO_MAP_CATALOG),
|
||||
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
|
||||
},
|
||||
"map": map_contract,
|
||||
}
|
||||
return {name: contracts[name] for name in sorted(required)}
|
||||
|
||||
@@ -279,6 +306,8 @@ def _prepare(
|
||||
# usable 16px-or-larger starting point.
|
||||
template_html = _apply_minimum_font_size(source.read_text(encoding="utf-8"))
|
||||
template_html = _apply_document_theme(template_html, profile.theme)
|
||||
runtime_base_url = _html_runtime_base_url()
|
||||
template_html = template_html.replace(_RUNTIME_BASE_PLACEHOLDER, runtime_base_url)
|
||||
working_copy.write_text(template_html, encoding="utf-8")
|
||||
brief = {
|
||||
"template": profile.to_dict(),
|
||||
@@ -527,10 +556,20 @@ def _append_missing_contract_errors(errors: list[str], lowered: str, kind: str)
|
||||
("df-chart", "df-chart 图表容器"),
|
||||
),
|
||||
"map": (
|
||||
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-geo", "data-df-geo JSON 数据块"),
|
||||
("df-geo-map", "df-geo-map 地图容器"),
|
||||
(
|
||||
("leaflet.css", "Leaflet 样式 leaflet.css"),
|
||||
("leaflet.js", "Leaflet 浏览器运行库 leaflet.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-map", "data-df-map JSON 数据块"),
|
||||
('class="df-map"', "df-map 地图容器"),
|
||||
)
|
||||
if _active_html_map_engine() == "leaflet"
|
||||
else (
|
||||
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-geo", "data-df-geo JSON 数据块"),
|
||||
("df-geo-map", "df-geo-map 地图容器"),
|
||||
)
|
||||
),
|
||||
}
|
||||
labels = {"flowchart": "流程图", "chart": "图表", "map": "地图"}
|
||||
@@ -788,6 +827,78 @@ def _chart_errors(html: str) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def _plain_description_errors(map_id: str, data: dict) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for key in ("regions", "markers", "lines", "polygons"):
|
||||
if key in data and not isinstance(data[key], list):
|
||||
errors.append(f"地图 {map_id} 的 {key} 必须是数组。")
|
||||
continue
|
||||
for item in data.get(key, []):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
description = item.get("description")
|
||||
if isinstance(description, str) and re.search(
|
||||
r"<\s*/?\s*[a-zA-Z][^>]*>|\bon\w+\s*=",
|
||||
description,
|
||||
):
|
||||
errors.append(f"地图 {map_id} 的 description 必须是纯文本,不能包含 HTML 或事件属性。")
|
||||
break
|
||||
return errors
|
||||
|
||||
|
||||
def _leaflet_map_errors(html: str) -> list[str]:
|
||||
"""Validate the Leaflet map-webpage contract without loading a browser."""
|
||||
errors: list[str] = []
|
||||
pattern = re.compile(
|
||||
r"<script\b[^>]*data-df-map\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
for map_id, payload in pattern.findall(html):
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
errors.append(f"地图 {map_id} 的 data-df-map 不是严格 JSON。")
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
errors.append(f"地图 {map_id} 的 data-df-map 必须是 JSON 对象。")
|
||||
continue
|
||||
for marker in data.get("markers") or []:
|
||||
if not isinstance(marker, dict):
|
||||
continue
|
||||
coordinate = marker.get("coordinate")
|
||||
if not isinstance(coordinate, list) or len(coordinate) < 2:
|
||||
errors.append(f"地图 {map_id} 的标点 {marker.get('name') or ''} 缺少 [经度,纬度]。")
|
||||
continue
|
||||
try:
|
||||
lng, lat = float(coordinate[0]), float(coordinate[1])
|
||||
except (TypeError, ValueError):
|
||||
errors.append(f"地图 {map_id} 的标点坐标必须是数字。")
|
||||
continue
|
||||
if not (-180 <= lng <= 180 and -90 <= lat <= 90):
|
||||
errors.append(f"地图 {map_id} 的标点坐标超出经纬度范围。")
|
||||
errors.extend(_plain_description_errors(map_id, data))
|
||||
return errors
|
||||
|
||||
|
||||
def _map_contract_errors(html: str) -> list[str]:
|
||||
"""Reject the map engine that the current setting does not allow."""
|
||||
lowered = html.lower()
|
||||
errors: list[str] = []
|
||||
if _active_html_map_engine() == "leaflet":
|
||||
if any(marker in lowered for marker in ("data-df-geo", "df-geo-map", "echarts.registermap")):
|
||||
errors.append("当前 HTML 地图引擎是地图网页,请使用 Leaflet 的 data-df-map,不要使用 data-df-geo。")
|
||||
if re.search(r"\bl\s*\.\s*map\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 Leaflet;请只提供 data-df-map JSON 数据块。")
|
||||
errors.extend(_leaflet_map_errors(html))
|
||||
return errors
|
||||
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
|
||||
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
|
||||
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
|
||||
errors.extend(_geo_map_errors(html))
|
||||
return errors
|
||||
|
||||
|
||||
def _quality_report(
|
||||
html: str,
|
||||
*,
|
||||
@@ -826,11 +937,7 @@ def _quality_report(
|
||||
errors.extend(_chart_errors(html))
|
||||
errors.extend(_dark_component_contrast_errors(html))
|
||||
if "map" in required_visualizations:
|
||||
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
|
||||
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
|
||||
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
|
||||
errors.extend(_geo_map_errors(html))
|
||||
errors.extend(_map_contract_errors(html))
|
||||
if expected_layout_mode == "document":
|
||||
if re.search(r"(?:html\s*,\s*body|body|html)\s*\{[^}]*\boverflow\s*:\s*hidden", lowered):
|
||||
errors.append("普通单页报告必须允许纵向滚动,禁止对 html/body 设置 overflow:hidden。")
|
||||
|
||||
+39
-10
@@ -208,8 +208,11 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
factory = get_session_factory()
|
||||
if factory is None:
|
||||
return None
|
||||
store = make_skill_store(factory)
|
||||
records = run_db_blocking(store.list_visible(user_id))
|
||||
async def read_visible():
|
||||
store = make_skill_store(get_session_factory())
|
||||
return await store.list_visible(user_id)
|
||||
|
||||
records = run_db_blocking(read_visible())
|
||||
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
|
||||
except Exception:
|
||||
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
|
||||
@@ -236,10 +239,18 @@ def skill_view_impl(name: str, agent_id: str | None, extra_skills: list[str] | N
|
||||
|
||||
storage = get_or_new_skill_storage()
|
||||
try:
|
||||
# Discover through the enabled catalog instead of reconstructing a
|
||||
# path from ``name``. Besides enforcing the online/offline switch,
|
||||
# this preserves skills whose directory differs from frontmatter name.
|
||||
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
|
||||
# A warm catalogue already knows each skill's file. Read that one file.
|
||||
# A cold catalogue checks the canonical directory before walking every
|
||||
# SKILL.md, which is only needed for a directory whose name differs.
|
||||
from deerflow.agents.lead_agent.prompt import _load_named_enabled_skill, get_warm_enabled_skills
|
||||
|
||||
cached = get_warm_enabled_skills(storage)
|
||||
if cached is not None:
|
||||
skill = next((item for item in cached if item.name == name), None)
|
||||
else:
|
||||
skill = _load_named_enabled_skill(name)
|
||||
if skill is None:
|
||||
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
|
||||
if skill is None:
|
||||
return f"Skill '{name}' not found."
|
||||
content = skill.skill_file.read_text(encoding="utf-8")
|
||||
@@ -320,7 +331,7 @@ def search_skills_impl(query: str, agent_id: str | None, top_n: int = 5, extra_s
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
skills = _skills_for_catalogue(storage)
|
||||
usage = all_entries()
|
||||
base_path = _resolve_skills_container_base_path()
|
||||
scored: list[tuple[int, dict]] = []
|
||||
@@ -396,11 +407,29 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
return skill_list_impl(_resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
|
||||
|
||||
|
||||
def _latest_activity(entry: dict) -> str | None:
|
||||
candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
|
||||
valid = [ts for ts in candidates if ts]
|
||||
if valid:
|
||||
return max(valid)
|
||||
return entry.get("created_at")
|
||||
|
||||
|
||||
def _skills_for_catalogue(storage):
|
||||
"""Use the warm prompt catalogue, and parse SKILL.md files only when it is cold."""
|
||||
from deerflow.agents.lead_agent.prompt import get_warm_enabled_skills
|
||||
|
||||
cached = get_warm_enabled_skills(storage)
|
||||
if cached is not None:
|
||||
return cached
|
||||
return storage.load_skills(enabled_only=True)
|
||||
|
||||
|
||||
def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None) -> str:
|
||||
"""List skills visible to ``agent_id`` (None = default lead agent)."""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
|
||||
|
||||
allowed = _load_agent_skill_whitelist(agent_id, extra_skills)
|
||||
|
||||
@@ -413,7 +442,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
skills = _skills_for_catalogue(storage)
|
||||
usage = all_entries()
|
||||
result = []
|
||||
for skill in skills:
|
||||
@@ -440,7 +469,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
|
||||
"use_count": entry.get("use_count", 0),
|
||||
"view_count": entry.get("view_count", 0),
|
||||
"patch_count": entry.get("patch_count", 0),
|
||||
"last_activity_at": latest_activity_at(skill.name),
|
||||
"last_activity_at": _latest_activity(entry),
|
||||
"pinned": entry.get("pinned", False),
|
||||
"source": entry.get("source", "upload"),
|
||||
})
|
||||
|
||||
+8
-2
@@ -147,7 +147,7 @@ async def task_tool(
|
||||
|
||||
# Get available tools (excluding task tool to prevent nesting)
|
||||
# Lazy import to avoid circular dependency
|
||||
from deerflow.tools import get_available_tools
|
||||
from deerflow.tools import get_available_tools_async
|
||||
|
||||
# Inherit parent agent's tool_groups so subagents respect the same restrictions
|
||||
parent_tool_groups = metadata.get("tool_groups")
|
||||
@@ -157,6 +157,8 @@ async def task_tool(
|
||||
effective_model = resolve_subagent_model_name(config, parent_model, app_config=resolved_app_config)
|
||||
|
||||
# Subagents should not have subagent tools enabled (prevent recursive nesting)
|
||||
parent_context = runtime.context if runtime is not None and isinstance(getattr(runtime, "context", None), dict) else {}
|
||||
roundtable_file_stream = bool(parent_context.get("roundtable_write_file_compatibility"))
|
||||
available_tools_kwargs = {
|
||||
"model_name": effective_model,
|
||||
"groups": parent_tool_groups,
|
||||
@@ -164,7 +166,9 @@ async def task_tool(
|
||||
}
|
||||
if resolved_app_config is not None:
|
||||
available_tools_kwargs["app_config"] = resolved_app_config
|
||||
tools = get_available_tools(**available_tools_kwargs)
|
||||
if roundtable_file_stream:
|
||||
available_tools_kwargs["write_file_compatibility"] = True
|
||||
tools = await get_available_tools_async(**available_tools_kwargs)
|
||||
|
||||
# Create executor
|
||||
executor_kwargs = {
|
||||
@@ -178,6 +182,8 @@ async def task_tool(
|
||||
}
|
||||
if resolved_app_config is not None:
|
||||
executor_kwargs["app_config"] = resolved_app_config
|
||||
if roundtable_file_stream:
|
||||
executor_kwargs["write_file_compatibility"] = True
|
||||
executor = SubagentExecutor(**executor_kwargs)
|
||||
|
||||
# Start background execution (always async to prevent blocking)
|
||||
|
||||
+113
-10
@@ -4,13 +4,14 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langgraph.typing import ContextT
|
||||
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.agents.lead_agent.prompt import refresh_one_skill_prompt_cache
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.mcp.tools import _make_sync_tool_wrapper
|
||||
from deerflow.persistence.skills import SkillStore
|
||||
@@ -66,6 +67,100 @@ async def _to_thread(func, /, *args, **kwargs):
|
||||
return await asyncio.to_thread(func, *args, **kwargs)
|
||||
|
||||
|
||||
_BLANK_SKILL_CONTENT = frozenset({"", "null", "none"})
|
||||
_SKILL_SOURCE_BYTES = 512 * 1024
|
||||
|
||||
|
||||
def _content_is_blank(content: str | None) -> bool:
|
||||
return content is None or content.strip().lower() in _BLANK_SKILL_CONTENT
|
||||
|
||||
|
||||
def _content_required_message(action: str, name: str) -> str:
|
||||
location = f"/mnt/user-data/workspace/{name}/SKILL.md"
|
||||
return (
|
||||
f"content is required for {action} and was empty or null. "
|
||||
f"Write the complete SKILL.md with write_file to {location}, then call "
|
||||
f'skill_manage(action="{action}", name="{name}", path="{location}"). '
|
||||
"Do not call this tool again with content set to null."
|
||||
)
|
||||
|
||||
|
||||
def _workspace_markdown_file(runtime: ToolRuntime[ContextT, ThreadState] | None, virtual_path: str) -> Path:
|
||||
"""Map a thread virtual path to a host file and reject traversal."""
|
||||
from deerflow.sandbox.tools import get_thread_data, replace_virtual_path
|
||||
|
||||
if not virtual_path.startswith("/mnt/user-data/"):
|
||||
raise ValueError("path must be under /mnt/user-data/workspace.")
|
||||
thread_data = get_thread_data(runtime)
|
||||
if not thread_data:
|
||||
raise ValueError("No thread workspace is available.")
|
||||
host = replace_virtual_path(virtual_path, thread_data)
|
||||
if host == virtual_path:
|
||||
raise ValueError("path must be under /mnt/user-data/workspace.")
|
||||
resolved = Path(host).resolve()
|
||||
for key in ("workspace_path", "uploads_path", "outputs_path"):
|
||||
base = thread_data.get(key)
|
||||
if not base:
|
||||
continue
|
||||
root = Path(base).resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError:
|
||||
continue
|
||||
return resolved
|
||||
raise ValueError("path must stay inside the thread workspace.")
|
||||
|
||||
|
||||
async def _read_workspace_markdown(runtime: ToolRuntime[ContextT, ThreadState] | None, virtual_path: str) -> str:
|
||||
target = _workspace_markdown_file(runtime, virtual_path)
|
||||
|
||||
def _read() -> str:
|
||||
if not target.is_file():
|
||||
raise ValueError(f"Skill file not found at {virtual_path}.")
|
||||
data = target.read_bytes()
|
||||
if len(data) > _SKILL_SOURCE_BYTES:
|
||||
raise ValueError("Skill file is too large to import.")
|
||||
return data.decode("utf-8")
|
||||
|
||||
return await _to_thread(_read)
|
||||
|
||||
|
||||
async def _resolve_skill_markdown(
|
||||
runtime: ToolRuntime[ContextT, ThreadState] | None,
|
||||
name: str,
|
||||
content: str | None,
|
||||
path: str | None,
|
||||
*,
|
||||
action: str,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Return markdown, or an instruction when the model omitted the body."""
|
||||
if not _content_is_blank(content):
|
||||
return content, None
|
||||
source = (path or "").strip()
|
||||
if source:
|
||||
try:
|
||||
return await _read_workspace_markdown(runtime, source), None
|
||||
except ValueError as exc:
|
||||
return None, f"{exc} {_content_required_message(action, name)}"
|
||||
return None, _content_required_message(action, name)
|
||||
|
||||
|
||||
async def _sync_managed_skill_view(name: str) -> None:
|
||||
"""Copy or remove this skill in the sandbox view. A cold view still rebuilds once."""
|
||||
try:
|
||||
from deerflow.sandbox.skill_view import sync_skill_sandbox_skill
|
||||
|
||||
await _to_thread(sync_skill_sandbox_skill, name)
|
||||
except Exception:
|
||||
logger.debug("Failed to sync sandbox view for skill %r", name, exc_info=True)
|
||||
|
||||
|
||||
async def _publish_managed_skill(name: str, *, removed: bool = False) -> None:
|
||||
"""Refresh the prompt entry and sandbox view for this skill only."""
|
||||
await refresh_one_skill_prompt_cache(name, removed=removed)
|
||||
await _sync_managed_skill_view(name)
|
||||
|
||||
|
||||
async def _bump_patch_best_effort(name: str) -> None:
|
||||
"""记一次技能维护活动(patch / edit)。
|
||||
|
||||
@@ -240,10 +335,15 @@ async def _skill_manage_impl(
|
||||
|
||||
async with lock:
|
||||
if action == "create":
|
||||
if await _to_thread(skill_storage.custom_skill_exists, name):
|
||||
# The writer always creates custom/<name>/. Stat that file instead
|
||||
# of parsing every SKILL.md. Edit and delete still resolve a
|
||||
# directory whose name differs from its frontmatter.
|
||||
skill_file = skill_storage.get_custom_skill_file(name)
|
||||
if await _to_thread(skill_file.is_file):
|
||||
raise ValueError(f"Custom skill '{name}' already exists.")
|
||||
content, hint = await _resolve_skill_markdown(runtime, name, content, path, action="create")
|
||||
if content is None:
|
||||
raise ValueError("content is required for create.")
|
||||
return hint or _content_required_message("create", name)
|
||||
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
|
||||
scan = await _scan_or_raise(content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
|
||||
await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, content)
|
||||
@@ -263,7 +363,7 @@ async def _skill_manage_impl(
|
||||
# async stack — contextvars don't propagate into ``to_thread``.
|
||||
if skill_store is not None:
|
||||
await _ensure_skill_db_registered(skill_store, name, user_id)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
await _publish_managed_skill(name)
|
||||
return f"Created custom skill '{name}'."
|
||||
|
||||
if action == "edit":
|
||||
@@ -271,8 +371,9 @@ async def _skill_manage_impl(
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
content, hint = await _resolve_skill_markdown(runtime, name, content, path, action="edit")
|
||||
if content is None:
|
||||
raise ValueError("content is required for edit.")
|
||||
return hint or _content_required_message("edit", name)
|
||||
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
|
||||
scan = await _scan_or_raise(content, executable=False, location=f"{name}/{SKILL_MD_FILE}")
|
||||
skill_file = skill_storage.get_custom_skill_file(name)
|
||||
@@ -284,7 +385,7 @@ async def _skill_manage_impl(
|
||||
_history_record(action="edit", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _bump_patch_best_effort(name)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
await _publish_managed_skill(name)
|
||||
return f"Updated custom skill '{name}'."
|
||||
|
||||
if action == "patch":
|
||||
@@ -312,7 +413,7 @@ async def _skill_manage_impl(
|
||||
_history_record(action="patch", file_path=SKILL_MD_FILE, prev_content=prev_content, new_content=new_content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _bump_patch_best_effort(name)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
await _publish_managed_skill(name)
|
||||
return f"Patched custom skill '{name}' ({replacement_count} replacement(s) applied, {occurrences} match(es) found)."
|
||||
|
||||
if action == "delete":
|
||||
@@ -366,7 +467,7 @@ async def _skill_manage_impl(
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Cascade failed for deleted skill %r", name, exc_info=True)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
await _publish_managed_skill(name, removed=True)
|
||||
return f"Deleted custom skill '{name}'."
|
||||
|
||||
if action == "write_file":
|
||||
@@ -387,6 +488,7 @@ async def _skill_manage_impl(
|
||||
name,
|
||||
_history_record(action="write_file", file_path=path, prev_content=prev_content, new_content=content, thread_id=thread_id, scanner=scan),
|
||||
)
|
||||
await _sync_managed_skill_view(name)
|
||||
return f"Wrote '{path}' for custom skill '{name}'."
|
||||
|
||||
if action == "remove_file":
|
||||
@@ -406,6 +508,7 @@ async def _skill_manage_impl(
|
||||
name,
|
||||
_history_record(action="remove_file", file_path=path, prev_content=prev_content, new_content=None, thread_id=thread_id, scanner={"decision": "allow", "reason": "Deletion requested."}),
|
||||
)
|
||||
await _sync_managed_skill_view(name)
|
||||
return f"Removed '{path}' from custom skill '{name}'."
|
||||
|
||||
if await _to_thread(skill_storage.public_skill_exists, name):
|
||||
@@ -429,8 +532,8 @@ async def skill_manage_tool(
|
||||
Args:
|
||||
action: One of create, patch, edit, delete, write_file, remove_file.
|
||||
name: Skill name in hyphen-case.
|
||||
content: New file content for create, edit, or write_file.
|
||||
path: Supporting file path for write_file or remove_file.
|
||||
content: Full SKILL.md text for create or edit, or file text for write_file. Never pass null. When the SKILL.md is long, omit content and pass path instead.
|
||||
path: Supporting file path for write_file or remove_file. For create or edit, a /mnt/user-data/workspace SKILL.md written first with write_file.
|
||||
find: Existing text to replace for patch.
|
||||
replace: Replacement text for patch.
|
||||
expected_count: Optional expected number of replacements for patch.
|
||||
|
||||
@@ -58,6 +58,41 @@ def _enforce_agent_orchestration_scope(
|
||||
if allow_agent_orchestration:
|
||||
return tools
|
||||
return [tool for tool in tools if tool.name != "agent_orchestration"]
|
||||
|
||||
|
||||
async def get_available_tools_async(
|
||||
groups: list[str] | None = None,
|
||||
include_mcp: bool = True,
|
||||
model_name: str | None = None,
|
||||
subagent_enabled: bool = False,
|
||||
is_scheduled_run: bool = False,
|
||||
excluded_tools: list[str] | None = None,
|
||||
*,
|
||||
allowed_tools: list[str] | None = None,
|
||||
allow_agent_orchestration: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
write_file_compatibility: bool | None = None,
|
||||
) -> list[BaseTool]:
|
||||
"""Load tools without blocking the caller loop or its MCP discovery.
|
||||
|
||||
Share bounded initialization with agent construction and restore the
|
||||
resulting deferred-tool registry into the calling task before execution.
|
||||
"""
|
||||
from deerflow.utils.background_init import run_background_init
|
||||
|
||||
return await run_background_init(
|
||||
get_available_tools,
|
||||
groups=groups,
|
||||
include_mcp=include_mcp,
|
||||
model_name=model_name,
|
||||
subagent_enabled=subagent_enabled,
|
||||
is_scheduled_run=is_scheduled_run,
|
||||
excluded_tools=excluded_tools,
|
||||
allowed_tools=allowed_tools,
|
||||
allow_agent_orchestration=allow_agent_orchestration,
|
||||
app_config=app_config,
|
||||
write_file_compatibility=write_file_compatibility,
|
||||
)
|
||||
|
||||
|
||||
def get_available_tools(
|
||||
@@ -68,8 +103,10 @@ def get_available_tools(
|
||||
is_scheduled_run: bool = False,
|
||||
excluded_tools: list[str] | None = None,
|
||||
*,
|
||||
allowed_tools: list[str] | None = None,
|
||||
allow_agent_orchestration: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
write_file_compatibility: bool | None = None,
|
||||
) -> list[BaseTool]:
|
||||
"""Get all available tools from config.
|
||||
|
||||
@@ -84,6 +121,10 @@ def get_available_tools(
|
||||
excluded_tools: Optional list of tool names to exclude from the result
|
||||
(applied AFTER all other resolution). Takes precedence over
|
||||
``app_config.excluded_tools``.
|
||||
allowed_tools: Optional request-scoped allowlist applied after all tools
|
||||
have been resolved. ``None`` keeps the normal tool surface; an empty
|
||||
list disables every ordinary tool. Exclusions are applied afterwards
|
||||
and therefore still win.
|
||||
allow_agent_orchestration: Explicit coordinator-only capability gate. Defaults to
|
||||
``False`` so ordinary agents and all indirect callers fail closed.
|
||||
|
||||
@@ -100,6 +141,8 @@ def get_available_tools(
|
||||
# requests entirely inside PatchedChatOpenAI.
|
||||
global_write_file_compatibility_enabled = (
|
||||
get_write_file_compatibility_settings().enabled
|
||||
if write_file_compatibility is None
|
||||
else write_file_compatibility
|
||||
)
|
||||
write_renderer_enabled = (
|
||||
not global_write_file_compatibility_enabled
|
||||
@@ -220,9 +263,15 @@ def get_available_tools(
|
||||
|
||||
local_wiki = getattr(getattr(config.llmwiki, "local_wiki_index", None), "enabled", False)
|
||||
if get_resolved_llmwiki_runtime(config).weknora_enabled or local_wiki or get_assistant_knowledge_store() is not None:
|
||||
from deerflow.tools.builtins.llmwiki_reasoning_tools import (
|
||||
CLASSIC_LLMWIKI_TOOLS,
|
||||
SMART_REASONING_TOOLS,
|
||||
)
|
||||
from deerflow.tools.builtins.llmwiki_search_tool import llmwiki_search_tool
|
||||
|
||||
builtin_tools.append(llmwiki_search_tool)
|
||||
builtin_tools.extend(CLASSIC_LLMWIKI_TOOLS)
|
||||
builtin_tools.extend(SMART_REASONING_TOOLS)
|
||||
except Exception: # noqa: BLE001 - configuration/store failures must not block agent creation
|
||||
logger.debug("Failed to evaluate the conditional LLMWiki search tool", exc_info=True)
|
||||
|
||||
@@ -333,6 +382,22 @@ def get_available_tools(
|
||||
if not write_renderer_enabled:
|
||||
unique_tools = [tool for tool in unique_tools if tool.name != _BEGIN_WRITE_FILE_TOOL_NAME]
|
||||
|
||||
# Security-sensitive internal runs (notably the roundtable coordinator)
|
||||
# must fail closed when new configured/builtin/MCP tools are added. A
|
||||
# denylist cannot provide that guarantee: every new tool would otherwise be
|
||||
# exposed until each caller remembered to exclude it.
|
||||
if allowed_tools is not None:
|
||||
allowed_set = {str(name).strip() for name in allowed_tools if str(name).strip()}
|
||||
filtered_tools = [tool for tool in unique_tools if tool.name in allowed_set]
|
||||
if len(filtered_tools) < len(unique_tools):
|
||||
logger.info(
|
||||
"Allowed request tool surface contains %d of %d resolved tool(s): %s",
|
||||
len(filtered_tools),
|
||||
len(unique_tools),
|
||||
sorted(allowed_set),
|
||||
)
|
||||
unique_tools = filtered_tools
|
||||
|
||||
# Drop tools the caller (or config) wants excluded. Request-level
|
||||
# ``excluded_tools`` wins; the config-level list is the static fallback.
|
||||
effective_excluded = excluded_tools if excluded_tools else getattr(config, "excluded_tools", None) or []
|
||||
|
||||
Reference in new issue
Block a user