sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -31,6 +31,7 @@ from __future__ import annotations
import logging
import re
from collections.abc import Mapping
from typing import Annotated, Any
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
@@ -296,6 +297,21 @@ def strip_solution_numeric_citations(markdown: str) -> str:
return "".join(lines)
def _report_requires_unnumbered_sources(runtime_context: Mapping[str, Any]) -> bool:
"""Return whether this run's report must avoid numbered source markers."""
project_context = runtime_context.get("analysis_project")
project_mode = (
str(project_context.get("mode") or "").strip().lower()
if isinstance(project_context, Mapping)
else ""
)
retrieval_mode = str(
runtime_context.get("knowledge_retrieval_mode") or "classic"
).strip().lower()
return project_mode == "solution" or retrieval_mode == "smart_reasoning"
_WORKBENCH_CONFIG_KEYS = frozenset(
{
"custom_outline",
@@ -325,6 +341,7 @@ def build_writing_research_config(
project_mode: str | None = None,
project_report_config: dict | None = None,
workbench_report_config: dict | None = None,
unnumbered_sources: bool = False,
) -> DeepResearchConfig:
"""组装聊天写作模式的研究配置:basic + 通用分析结构 + adaptive。
@@ -336,6 +353,7 @@ def build_writing_research_config(
工作台确认卡传入 ``workbench_report_config`` 时沿用卡片上的大纲/结构/配图
等选项,但 **mode 仍强制 basic**:那是普通问答写作模式在脆弱服务器上能
写完的同一条 runner,避免再走 deep/multi_agent 的 job 路径。
``unnumbered_sources`` 只改变引用写法,不切换方案模式或固定大纲。
"""
parts: list[str] = []
if focus and focus.strip():
@@ -352,6 +370,8 @@ def build_writing_research_config(
if extra_instruction:
parts.append(extra_instruction)
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
if fixed_solution or unnumbered_sources:
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
return DeepResearchConfig(
mode="basic",
collection_mode="legacy",
@@ -380,6 +400,8 @@ def build_writing_research_config(
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
else:
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
if unnumbered_sources:
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
return DeepResearchConfig(
mode="basic",
collection_mode="legacy",
@@ -632,7 +654,7 @@ async def deep_research_report_tool(
The tool harvests the materials already produced in this conversation (search /
knowledge / skill / file tool results) and writes a full research report
(general-analysis structure, numbered reference section) to
(general-analysis structure, with source provenance) to
/mnt/user-data/outputs/report.md in this thread. The file is presented
automatically; do NOT write the report body yourself with write_file.
@@ -691,7 +713,11 @@ async def deep_research_report_tool(
workbench_report_config = _coerce_workbench_report_config(runtime_context.get("writing_report_config"))
workbench_session_id = str(runtime_context.get("workbench_session_id") or "").strip()
force_no_materials = bool(runtime_context.get("force_no_materials"))
harvest_options = {"include_rag_sources": True} if project_context.get("mode") == "solution" else {}
harvest_options = (
{"include_rag_sources": True}
if project_context.get("mode") == "solution" or workbench_session_id
else {}
)
if force_no_materials:
harvested, dropped_count, harvested_count = [], 0, 0
materials: list[ResearchMaterial] = []
@@ -742,6 +768,7 @@ async def deep_research_report_tool(
project_mode=project_context.get("mode"),
project_report_config=project_context.get("report_config"),
workbench_report_config=workbench_report_config or None,
unnumbered_sources=_report_requires_unnumbered_sources(runtime_context),
)
if workbench_session_id:
await _persist_workbench_session(
@@ -781,7 +808,7 @@ async def deep_research_report_tool(
error_message: str | None = None
try:
generated_result = await DeepResearchEngine().run(request, adapters=bundle)
if project_context.get("mode") == "solution":
if _report_requires_unnumbered_sources(runtime_context):
cleaned_report = strip_solution_numeric_citations(generated_result.report_markdown)
if cleaned_report != generated_result.report_markdown:
await bundle.artifacts.write_text("report.md", cleaned_report)
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Mapping
@@ -98,6 +99,42 @@ def _sanitize_conversation_deposit_value(value: Any) -> Any:
return value
def _stable_citation_id(item: Mapping[str, Any]) -> str:
"""Build a stable cross-call identity without exposing provider IDs."""
knowledge_base_id = str(
item.get("knowledge_base_id") or item.get("base_id") or "unknown"
)
identity = next(
(
str(item.get(key) or "").strip()
for key in (
"chunk_id",
"page_ref",
"wiki_slug",
"slug",
"knowledge_id",
"id",
"title",
)
if str(item.get(key) or "").strip()
),
"result",
)
digest = hashlib.sha256(f"{knowledge_base_id}\0{identity}".encode()).hexdigest()[:20]
return f"llmwiki:{digest}"
def _result_score(item: Mapping[str, Any]) -> float:
for key in ("relevance_score", "rerank_score", "score", "similarity"):
try:
value = float(item.get(key))
except (TypeError, ValueError):
continue
return value
return 0.0
def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
"""Read the conversation selection; ``None`` means legacy/no selection key."""
@@ -120,7 +157,7 @@ def selected_assistant_knowledge_base_ids(context: Mapping[str, Any] | None) ->
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
agent_id = str((context or {}).get("agent_id") or "").strip()
if not agent_id or agent_id in {"default", "lead_agent"}:
if not agent_id or agent_id in {"default", "lead_agent", "deep-research-planner", "deep-research-collector"}:
return None
try:
config = load_agent_config(agent_id)
@@ -133,6 +170,7 @@ def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
async def llmwiki_search_tool(
query: str,
runtime: ToolRuntime[ContextT, ThreadState],
knowledge_base_id: str = "",
) -> str:
"""Search the LLMWiki knowledge selected for this conversation.
@@ -142,6 +180,9 @@ async def llmwiki_search_tool(
Args:
query: Focused natural-language retrieval query.
knowledge_base_id: Optional selected DeerFlow knowledge-base ID. Use one
ID per call when several libraries are selected so each library is
searched and diagnosed independently.
"""
started_at = perf_counter()
@@ -153,9 +194,36 @@ async def llmwiki_search_tool(
context = runtime.context if runtime.context else {}
selected_ids = selected_knowledge_base_ids(context)
assistant_selected_ids = selected_assistant_knowledge_base_ids(context)
allowed_ids = _agent_allowed_ids(context)
selected_ids = selected_ids or []
assistant_selected_ids = assistant_selected_ids or []
retrieval_mode = str(context.get("knowledge_retrieval_mode") or "classic").strip().lower()
if retrieval_mode != "classic":
return _trace_json(
started_at=started_at,
query=query,
selection=selected_ids,
assistant_selection=assistant_selected_ids,
zero_result_reason="CLASSIC_MODE_REQUIRED",
error="llmwiki_search is available only in classic knowledge retrieval mode.",
)
allowed_ids = _agent_allowed_ids(context)
requested_knowledge_base_id = str(knowledge_base_id or "").strip()
if requested_knowledge_base_id:
if requested_knowledge_base_id in selected_ids:
selected_ids = [requested_knowledge_base_id]
assistant_selected_ids = []
elif requested_knowledge_base_id in assistant_selected_ids:
selected_ids = []
assistant_selected_ids = [requested_knowledge_base_id]
else:
return _trace_json(
started_at=started_at,
query=query,
selection=selected_ids,
assistant_selection=assistant_selected_ids,
zero_result_reason="KNOWLEDGE_SCOPE_DENIED",
error="knowledge_base_id is not selected for this conversation.",
)
if not selected_ids and not assistant_selected_ids:
return _trace_json(
started_at=started_at,
@@ -323,6 +391,11 @@ async def llmwiki_search_tool(
else:
mode = str(payload.get("mode") or "empty")
results.sort(key=_result_score, reverse=True)
for rank, item in enumerate(results, start=1):
item.setdefault("citation_id", _stable_citation_id(item))
item.setdefault("relevance_rank", rank)
if results:
zero_result_reason = None
elif fallback_reason:
@@ -62,7 +62,40 @@ def _required_visualizations(features: Iterable[str], material_brief: str = "")
return required
def _active_html_map_engine() -> str:
from deerflow.html_runtime.map_engine import html_map_engine
return html_map_engine()
_RUNTIME_BASE_PLACEHOLDER = "__DF_HTML_RUNTIME_BASE__"
def _html_runtime_base_url() -> str:
"""Browser-reachable runtime base saved in the settings dialog."""
from deerflow.html_runtime.addresses import html_runtime_base_url
return html_runtime_base_url()
def _visualization_contract(required: set[str]) -> dict[str, object]:
leaflet = _active_html_map_engine() == "leaflet"
map_contract: dict[str, object] = (
{
"renderer": "Leaflet 地图网页",
"required_assets": ["leaflet.css", "leaflet.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-map\"", "data-df-map"],
"forbidden": ["data-df-geo", "df-geo-map", "manual L.map", "tile URL in HTML"],
}
if leaflet
else {
"renderer": "ECharts GeoJSON(Canvas)",
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
"supported_maps": sorted(_GEO_MAP_CATALOG),
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
}
)
contracts: dict[str, dict[str, object]] = {
"flowchart": {
"renderer": "AntV G6",
@@ -75,13 +108,7 @@ def _visualization_contract(required: set[str]) -> dict[str, object]:
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-chart\"", "data-df-chart"],
},
"map": {
"renderer": "ECharts GeoJSON(Canvas)",
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
"supported_maps": sorted(_GEO_MAP_CATALOG),
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
},
"map": map_contract,
}
return {name: contracts[name] for name in sorted(required)}
@@ -279,6 +306,8 @@ def _prepare(
# usable 16px-or-larger starting point.
template_html = _apply_minimum_font_size(source.read_text(encoding="utf-8"))
template_html = _apply_document_theme(template_html, profile.theme)
runtime_base_url = _html_runtime_base_url()
template_html = template_html.replace(_RUNTIME_BASE_PLACEHOLDER, runtime_base_url)
working_copy.write_text(template_html, encoding="utf-8")
brief = {
"template": profile.to_dict(),
@@ -527,10 +556,20 @@ def _append_missing_contract_errors(errors: list[str], lowered: str, kind: str)
("df-chart", "df-chart 图表容器"),
),
"map": (
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-geo", "data-df-geo JSON 数据块"),
("df-geo-map", "df-geo-map 地图容器"),
(
("leaflet.css", "Leaflet 样式 leaflet.css"),
("leaflet.js", "Leaflet 浏览器运行库 leaflet.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-map", "data-df-map JSON 数据块"),
('class="df-map"', "df-map 地图容器"),
)
if _active_html_map_engine() == "leaflet"
else (
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-geo", "data-df-geo JSON 数据块"),
("df-geo-map", "df-geo-map 地图容器"),
)
),
}
labels = {"flowchart": "流程图", "chart": "图表", "map": "地图"}
@@ -788,6 +827,78 @@ def _chart_errors(html: str) -> list[str]:
return errors
def _plain_description_errors(map_id: str, data: dict) -> list[str]:
errors: list[str] = []
for key in ("regions", "markers", "lines", "polygons"):
if key in data and not isinstance(data[key], list):
errors.append(f"地图 {map_id} 的 {key} 必须是数组。")
continue
for item in data.get(key, []):
if not isinstance(item, dict):
continue
description = item.get("description")
if isinstance(description, str) and re.search(
r"<\s*/?\s*[a-zA-Z][^>]*>|\bon\w+\s*=",
description,
):
errors.append(f"地图 {map_id} 的 description 必须是纯文本,不能包含 HTML 或事件属性。")
break
return errors
def _leaflet_map_errors(html: str) -> list[str]:
"""Validate the Leaflet map-webpage contract without loading a browser."""
errors: list[str] = []
pattern = re.compile(
r"<script\b[^>]*data-df-map\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
flags=re.IGNORECASE,
)
for map_id, payload in pattern.findall(html):
try:
data = json.loads(payload)
except json.JSONDecodeError:
errors.append(f"地图 {map_id} 的 data-df-map 不是严格 JSON。")
continue
if not isinstance(data, dict):
errors.append(f"地图 {map_id} 的 data-df-map 必须是 JSON 对象。")
continue
for marker in data.get("markers") or []:
if not isinstance(marker, dict):
continue
coordinate = marker.get("coordinate")
if not isinstance(coordinate, list) or len(coordinate) < 2:
errors.append(f"地图 {map_id} 的标点 {marker.get('name') or ''} 缺少 [经度,纬度]。")
continue
try:
lng, lat = float(coordinate[0]), float(coordinate[1])
except (TypeError, ValueError):
errors.append(f"地图 {map_id} 的标点坐标必须是数字。")
continue
if not (-180 <= lng <= 180 and -90 <= lat <= 90):
errors.append(f"地图 {map_id} 的标点坐标超出经纬度范围。")
errors.extend(_plain_description_errors(map_id, data))
return errors
def _map_contract_errors(html: str) -> list[str]:
"""Reject the map engine that the current setting does not allow."""
lowered = html.lower()
errors: list[str] = []
if _active_html_map_engine() == "leaflet":
if any(marker in lowered for marker in ("data-df-geo", "df-geo-map", "echarts.registermap")):
errors.append("当前 HTML 地图引擎是地图网页,请使用 Leaflet 的 data-df-map,不要使用 data-df-geo。")
if re.search(r"\bl\s*\.\s*map\s*\(", lowered):
errors.append("不要在 HTML 中自行初始化 Leaflet;请只提供 data-df-map JSON 数据块。")
errors.extend(_leaflet_map_errors(html))
return errors
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
errors.extend(_geo_map_errors(html))
return errors
def _quality_report(
html: str,
*,
@@ -826,11 +937,7 @@ def _quality_report(
errors.extend(_chart_errors(html))
errors.extend(_dark_component_contrast_errors(html))
if "map" in required_visualizations:
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
errors.extend(_geo_map_errors(html))
errors.extend(_map_contract_errors(html))
if expected_layout_mode == "document":
if re.search(r"(?:html\s*,\s*body|body|html)\s*\{[^}]*\boverflow\s*:\s*hidden", lowered):
errors.append("普通单页报告必须允许纵向滚动,禁止对 html/body 设置 overflow:hidden。")
@@ -208,8 +208,11 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = run_db_blocking(store.list_visible(user_id))
async def read_visible():
store = make_skill_store(get_session_factory())
return await store.list_visible(user_id)
records = run_db_blocking(read_visible())
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
@@ -236,10 +239,18 @@ def skill_view_impl(name: str, agent_id: str | None, extra_skills: list[str] | N
storage = get_or_new_skill_storage()
try:
# Discover through the enabled catalog instead of reconstructing a
# path from ``name``. Besides enforcing the online/offline switch,
# this preserves skills whose directory differs from frontmatter name.
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
# A warm catalogue already knows each skill's file. Read that one file.
# A cold catalogue checks the canonical directory before walking every
# SKILL.md, which is only needed for a directory whose name differs.
from deerflow.agents.lead_agent.prompt import _load_named_enabled_skill, get_warm_enabled_skills
cached = get_warm_enabled_skills(storage)
if cached is not None:
skill = next((item for item in cached if item.name == name), None)
else:
skill = _load_named_enabled_skill(name)
if skill is None:
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
if skill is None:
return f"Skill '{name}' not found."
content = skill.skill_file.read_text(encoding="utf-8")
@@ -320,7 +331,7 @@ def search_skills_impl(query: str, agent_id: str | None, top_n: int = 5, extra_s
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
skills = _skills_for_catalogue(storage)
usage = all_entries()
base_path = _resolve_skills_container_base_path()
scored: list[tuple[int, dict]] = []
@@ -396,11 +407,29 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
return skill_list_impl(_resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
def _latest_activity(entry: dict) -> str | None:
candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
valid = [ts for ts in candidates if ts]
if valid:
return max(valid)
return entry.get("created_at")
def _skills_for_catalogue(storage):
"""Use the warm prompt catalogue, and parse SKILL.md files only when it is cold."""
from deerflow.agents.lead_agent.prompt import get_warm_enabled_skills
cached = get_warm_enabled_skills(storage)
if cached is not None:
return cached
return storage.load_skills(enabled_only=True)
def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None) -> str:
"""List skills visible to ``agent_id`` (None = default lead agent)."""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
allowed = _load_agent_skill_whitelist(agent_id, extra_skills)
@@ -413,7 +442,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
skills = _skills_for_catalogue(storage)
usage = all_entries()
result = []
for skill in skills:
@@ -440,7 +469,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
"use_count": entry.get("use_count", 0),
"view_count": entry.get("view_count", 0),
"patch_count": entry.get("patch_count", 0),
"last_activity_at": latest_activity_at(skill.name),
"last_activity_at": _latest_activity(entry),
"pinned": entry.get("pinned", False),
"source": entry.get("source", "upload"),
})
@@ -147,7 +147,7 @@ async def task_tool(
# Get available tools (excluding task tool to prevent nesting)
# Lazy import to avoid circular dependency
from deerflow.tools import get_available_tools
from deerflow.tools import get_available_tools_async
# Inherit parent agent's tool_groups so subagents respect the same restrictions
parent_tool_groups = metadata.get("tool_groups")
@@ -157,6 +157,8 @@ async def task_tool(
effective_model = resolve_subagent_model_name(config, parent_model, app_config=resolved_app_config)
# Subagents should not have subagent tools enabled (prevent recursive nesting)
parent_context = runtime.context if runtime is not None and isinstance(getattr(runtime, "context", None), dict) else {}
roundtable_file_stream = bool(parent_context.get("roundtable_write_file_compatibility"))
available_tools_kwargs = {
"model_name": effective_model,
"groups": parent_tool_groups,
@@ -164,7 +166,9 @@ async def task_tool(
}
if resolved_app_config is not None:
available_tools_kwargs["app_config"] = resolved_app_config
tools = get_available_tools(**available_tools_kwargs)
if roundtable_file_stream:
available_tools_kwargs["write_file_compatibility"] = True
tools = await get_available_tools_async(**available_tools_kwargs)
# Create executor
executor_kwargs = {
@@ -178,6 +182,8 @@ async def task_tool(
}
if resolved_app_config is not None:
executor_kwargs["app_config"] = resolved_app_config
if roundtable_file_stream:
executor_kwargs["write_file_compatibility"] = True
executor = SubagentExecutor(**executor_kwargs)
# Start background execution (always async to prevent blocking)