sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -174,12 +174,157 @@ _KNOWLEDGE_WRITE_INSTRUCTION = (
"写成报告主线或用其填充多个章节。参考文献可标为基于公开领域知识,不要用「材料未提供」占整节。"
)
_SOLUTION_REPORT_CITATION_INSTRUCTION = (
"最终方案正文不得出现 [1]、[2]、【1】、【2】等数字引用标记。"
"如需保留参考来源,请在参考来源小节中使用无编号项目符号,直接写来源名称和链接。"
)
_CITATION_DIGITS_PATTERN = r"[0-90-9]{1,3}"
_CITATION_SEQUENCE_PATTERN = (
rf"{_CITATION_DIGITS_PATTERN}"
rf"(?:\s*(?:[,,、;;]|[-–—~~])\s*{_CITATION_DIGITS_PATTERN})*"
)
_NUMERIC_CITATION_TOKEN_PATTERN = (
rf"(?:\[\[\s*{_CITATION_SEQUENCE_PATTERN}\s*\]\]"
rf"|[\[[【]\s*{_CITATION_SEQUENCE_PATTERN}\s*[\]]】])"
)
_NUMERIC_CITATION_TOKEN_RE = re.compile(_NUMERIC_CITATION_TOKEN_PATTERN)
_NUMERIC_CITATION_LINK_RE = re.compile(
rf"\[\s*{_CITATION_DIGITS_PATTERN}\s*\]"
r"\((?P<target>https?://[^)\s]+|#reference-[^)]+)\)"
)
_REFERENCE_ENTRY_PREFIX_RE = re.compile(
rf"^(?P<indent>\s*)(?:[-*+]\s*)?{_NUMERIC_CITATION_TOKEN_PATTERN}\s+(?=\S)"
)
_REFERENCE_LINK_ENTRY_PREFIX_RE = re.compile(
rf"^(?P<indent>\s*)(?:[-*+]\s*)?"
rf"\[\s*{_CITATION_DIGITS_PATTERN}\s*\]"
r"\((?P<target>https?://[^)\s]+|#reference-[^)]+)\)\s*"
)
_REFERENCE_DEFINITION_PREFIX_RE = re.compile(
rf"^(?P<indent>\s*)(?:[-*+]\s*)?"
rf"{_NUMERIC_CITATION_TOKEN_PATTERN}\s*:\s*"
)
_REFERENCE_HEADING_RE = re.compile(
r"^#{1,6}\s*(?:参考来源|参考文献|references|sources)\s*:?\s*$",
re.IGNORECASE,
)
_MARKDOWN_HEADING_RE = re.compile(r"^#{1,6}\s+\S")
_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
_INLINE_CODE_RE = re.compile(r"(`+[^`\n]*`+)")
def _strip_numeric_citations_from_prose(value: str) -> str:
"""Remove numeric citations while preserving inline-code examples."""
def _replace_numeric_link(match: re.Match[str]) -> str:
target = match.group("target")
return "" if target.startswith("#reference-") else f"<{target}>"
parts = _INLINE_CODE_RE.split(value)
for index in range(0, len(parts), 2):
prose = _NUMERIC_CITATION_LINK_RE.sub(_replace_numeric_link, parts[index])
parts[index] = _NUMERIC_CITATION_TOKEN_RE.sub("", prose)
return "".join(parts)
def strip_solution_numeric_citations(markdown: str) -> str:
"""Return a clean solution report without numbered citation markers.
Research sources remain useful in a solution project, so entries under a
reference heading are converted to ordinary unordered-list items instead
of being deleted. Fenced code is left untouched because ``[1]`` may be
valid Mermaid or source-code syntax there.
"""
if not markdown:
return markdown
lines: list[str] = []
in_fence = False
fence_char = ""
fence_width = 0
in_references = False
for line in markdown.splitlines(keepends=True):
raw_line = line.rstrip("\r\n")
newline = line[len(raw_line) :]
fence_match = _FENCE_RE.match(raw_line)
if fence_match:
marker = fence_match.group(1)
marker_char = marker[0]
if not in_fence:
in_fence = True
fence_char = marker_char
fence_width = len(marker)
elif marker_char == fence_char and len(marker) >= fence_width:
in_fence = False
fence_char = ""
fence_width = 0
lines.append(line)
continue
if in_fence:
lines.append(line)
continue
stripped = raw_line.strip()
if _REFERENCE_HEADING_RE.match(stripped):
in_references = True
lines.append(line)
continue
if in_references and _MARKDOWN_HEADING_RE.match(stripped):
in_references = False
cleaned = raw_line
if in_references:
cleaned = _REFERENCE_LINK_ENTRY_PREFIX_RE.sub(
lambda match: (
f"{match.group('indent')}- "
+ (f"<{match.group('target')}> " if not match.group("target").startswith("#") else "")
),
cleaned,
)
cleaned = _REFERENCE_DEFINITION_PREFIX_RE.sub(
lambda match: f"{match.group('indent')}- ",
cleaned,
)
cleaned = _REFERENCE_ENTRY_PREFIX_RE.sub(
lambda match: f"{match.group('indent')}- ",
cleaned,
)
cleaned = _strip_numeric_citations_from_prose(cleaned)
lines.append(cleaned + newline)
return "".join(lines)
_WORKBENCH_CONFIG_KEYS = frozenset(
{
"custom_outline",
"structure_mode",
"generate_images",
"max_generated_images",
"language",
"tone",
"report_instruction",
"retrieval_directions",
"retrieval_skills",
}
)
def _coerce_workbench_report_config(raw: Any) -> dict[str, Any]:
if not isinstance(raw, dict):
return {}
return {key: raw[key] for key in _WORKBENCH_CONFIG_KEYS if key in raw and raw[key] is not None}
def build_writing_research_config(
*,
focus: str | None = None,
thinking_enabled: bool = False,
model_name: str | None = None,
project_mode: str | None = None,
project_report_config: dict | None = None,
workbench_report_config: dict | None = None,
) -> DeepResearchConfig:
"""组装聊天写作模式的研究配置:basic + 通用分析结构 + adaptive。
@@ -187,16 +332,61 @@ def build_writing_research_config(
``model_name`` / ``thinking_enabled`` 继承自聊天 run 的模型选择——同一模型
覆盖 fast/smart/strategic 三个角色,使报告管线与对话用同一个模型和思考
开关(缺省回落系统默认模型)。
工作台确认卡传入 ``workbench_report_config`` 时沿用卡片上的大纲/结构/配图
等选项,但 **mode 仍强制 basic**:那是普通问答写作模式在脆弱服务器上能
写完的同一条 runner,避免再走 deep/multi_agent 的 job 路径。
"""
parts: list[str] = []
if focus and focus.strip():
parts.append(focus.strip())
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
from deerflow.agents.analysis_project import SOLUTION_OUTLINE
fixed_solution = project_mode == "solution"
project_report_config = project_report_config or {}
workbench = _coerce_workbench_report_config(workbench_report_config)
if workbench:
outline = str(workbench.get("custom_outline") or "").strip() or GENERAL_OUTLINE
structure_mode = workbench.get("structure_mode") or "adaptive"
extra_instruction = str(workbench.get("report_instruction") or "").strip()
if extra_instruction:
parts.append(extra_instruction)
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
return DeepResearchConfig(
mode="basic",
collection_mode="legacy",
custom_outline=outline,
structure_mode=structure_mode if structure_mode in ("fixed", "adaptive") else "adaptive",
retrieval_directions=list(workbench.get("retrieval_directions") or []),
retrieval_skills=list(workbench.get("retrieval_skills") or []),
generate_images=bool(workbench.get("generate_images")),
max_generated_images=workbench.get("max_generated_images") or 2,
language=str(workbench.get("language") or "zh-CN"),
tone=workbench.get("tone") or "objective",
thinking_enabled=thinking_enabled,
fast_model=model_name,
smart_model=model_name,
strategic_model=model_name,
report_instruction="\n".join(parts),
).clamp()
outline = project_report_config.get("custom_outline") or SOLUTION_OUTLINE
structure_mode = project_report_config.get("structure_mode", "fixed") if fixed_solution else "adaptive"
if fixed_solution:
parts.append(
"遵循项目所选报告结构及固定/弹性约束。引用本项目对话资料;"
"缺失事实、预算、责任人标注待补充,不得编造。"
)
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
else:
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
return DeepResearchConfig(
mode="basic",
collection_mode="legacy",
custom_outline=GENERAL_OUTLINE,
structure_mode="adaptive",
custom_outline=outline if fixed_solution else GENERAL_OUTLINE,
structure_mode=structure_mode,
retrieval_directions=project_report_config.get("retrieval_directions", []) if fixed_solution else [],
retrieval_skills=project_report_config.get("retrieval_skills", []) if fixed_solution else [],
generate_images=False,
thinking_enabled=thinking_enabled,
fast_model=model_name,
@@ -206,6 +396,54 @@ def build_writing_research_config(
).clamp()
async def _persist_workbench_session(
*,
session_id: str,
status: str,
report_markdown: str | None = None,
source_count: int | None = None,
error: str | None = None,
workbench_report_config: dict[str, Any] | None = None,
) -> None:
"""Best-effort session row update so refresh/follow-up still see the report.
Failures are logged only — the chat thread already has report.md in artifacts
and the custom-event stream already showed the live draft.
"""
session_id = str(session_id or "").strip()
if not session_id:
return
try:
from deerflow.persistence.deep_research_sessions import get_default_store
store = get_default_store()
if store is None:
return
fields: dict[str, Any] = {"status": status, "active_job_id": None, "error": error}
if report_markdown is not None:
fields["report_markdown"] = report_markdown
fields["report_html"] = None
if source_count is not None:
fields["source_count"] = source_count
if workbench_report_config:
row = await store.get(session_id, user_id=None)
snapshot = dict((row or {}).get("config_snapshot") or {})
snapshot.update(_coerce_workbench_report_config(workbench_report_config))
snapshot["mode"] = "basic"
snapshot["collection_mode"] = "legacy"
fields["config_snapshot"] = snapshot
# Session id is already scoped to this confirm-card write; skip the
# request user filter so a LangGraph tool task without contextvar
# still lands the completed report (the file card keys off this row).
await store.update(session_id, user_id=None, **fields)
except Exception: # noqa: BLE001 - 落库失败不能打断写作管线
logger.exception(
"[deep_research_report] workbench session persist failed session=%s status=%s",
session_id,
status,
)
def _resolve_chat_model(context: dict[str, Any] | None) -> tuple[str | None, bool, bool]:
"""从 run 上下文解析聊天选择的模型与思考开关(P2:管线继承聊天模型)。
@@ -434,20 +672,45 @@ async def deep_research_report_tool(
query = topic.strip()
state = getattr(runtime, "state", None) or {}
conversation_messages = state.get("messages") or []
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query)
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
anchor_index = topic_anchor_index(conversation_messages, topic=query)
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
harvested_count = len(materials)
logger.info(
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
thread_id,
len(harvested),
harvested_count,
dropped_count,
anchor_index,
)
runtime_context = runtime.context or {}
project_context = runtime_context.get("analysis_project") or {}
if project_context.get("mode") == "solution" and not runtime_context.get(
"solution_requirements_confirmed"
):
return Command(
update={
"messages": [
ToolMessage(
content=("当前仍处于正常问答阶段,不能生成首份方案。请继续自然理解用户意图;信息足够后输出简短的意图理解、合法意图快照及就绪标记,并等待用户点击‘结束问答并生成方案’后再调用本工具。"),
tool_call_id=tool_call_id,
name="deep_research_report",
)
]
}
)
workbench_report_config = _coerce_workbench_report_config(runtime_context.get("writing_report_config"))
workbench_session_id = str(runtime_context.get("workbench_session_id") or "").strip()
force_no_materials = bool(runtime_context.get("force_no_materials"))
harvest_options = {"include_rag_sources": True} if project_context.get("mode") == "solution" else {}
if force_no_materials:
harvested, dropped_count, harvested_count = [], 0, 0
materials: list[ResearchMaterial] = []
logger.info("[deep_research_report] thread=%s force_no_materials; writing with empty pool", thread_id)
else:
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query, **harvest_options)
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
anchor_index = topic_anchor_index(conversation_messages, topic=query)
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
harvested_count = len(materials)
logger.info(
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
thread_id,
len(harvested),
harvested_count,
dropped_count,
anchor_index,
)
if harvested_count < _MIN_HARVESTED_MATERIALS:
logger.info(
@@ -476,7 +739,16 @@ async def deep_research_report_tool(
focus=focus,
thinking_enabled=thinking_enabled,
model_name=model_name,
project_mode=project_context.get("mode"),
project_report_config=project_context.get("report_config"),
workbench_report_config=workbench_report_config or None,
)
if workbench_session_id:
await _persist_workbench_session(
session_id=workbench_session_id,
status="running",
workbench_report_config=workbench_report_config or None,
)
llm_backend = DeerFlowCompletionBackend(
research_config,
event_sink=events,
@@ -508,12 +780,36 @@ async def deep_research_report_tool(
result = None
error_message: str | None = None
try:
result = await DeepResearchEngine().run(request, adapters=bundle)
generated_result = await DeepResearchEngine().run(request, adapters=bundle)
if project_context.get("mode") == "solution":
cleaned_report = strip_solution_numeric_citations(generated_result.report_markdown)
if cleaned_report != generated_result.report_markdown:
await bundle.artifacts.write_text("report.md", cleaned_report)
generated_result = generated_result.model_copy(update={"report_markdown": cleaned_report})
result = generated_result
except Exception as exc: # noqa: BLE001 - 工具错误以 ToolMessage 回给模型自纠
logger.exception("[deep_research_report] thread=%s research failed", thread_id)
error_message = f"深度研究报告生成失败:{exc}"
# 终态帧:无论成败都发,前端据此收尾进度条 / 停止流式预览。
source_count = len(result.source_ids or []) if result is not None else 0
if workbench_session_id:
if result is None:
await _persist_workbench_session(
session_id=workbench_session_id,
status="failed",
error=error_message,
)
else:
await _persist_workbench_session(
session_id=workbench_session_id,
status="completed",
report_markdown=result.report_markdown or "",
source_count=max(source_count, harvested_count),
workbench_report_config=workbench_report_config or None,
)
# 终态帧放在落库之后:前端收到 tool_finished 再 refetch 时会话已是 completed,
# 消息流才能立刻插入 report.md 文件卡片。
if writer is not None:
try:
writer(
@@ -547,14 +843,16 @@ async def deep_research_report_tool(
}
)
source_count = len(result.source_ids or [])
summary = f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
logger.info(
"[deep_research_report] thread=%s completed (sources=%d, report_chars=%d)",
thread_id,
source_count,
len(result.report_markdown or ""),
)
summary = (
f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,"
f"已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
)
return Command(
update={
"artifacts": [REPORT_VIRTUAL_PATH],
@@ -5,7 +5,9 @@ from __future__ import annotations
import json
import re
from collections.abc import Mapping
from time import perf_counter
from typing import Any
from uuid import uuid4
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
@@ -13,7 +15,8 @@ from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
from deerflow.integrations.weknora.local_index.advanced import advanced_search
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_index_store, get_local_wiki_retrieval_service, get_local_wiki_search_service
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
@@ -23,6 +26,62 @@ _CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
def _trace_json(
*,
started_at: float,
query: str,
selection: list[str],
assistant_selection: list[str],
authorized_space_count: int = 0,
mode: str = "empty",
results: list[dict[str, Any]] | None = None,
partial: bool = False,
warnings: list[str] | None = None,
fallback_knowledge_base_ids: list[str] | None = None,
knowledge_base_profiles: list[dict[str, Any]] | None = None,
query_plan: dict[str, Any] | None = None,
statistics: dict[str, Any] | None = None,
references: list[dict[str, Any]] | None = None,
diagnostics: dict[str, Any] | None = None,
zero_result_reason: str | None = None,
error: str | None = None,
) -> str:
"""Serialize every tool outcome as a durable retrieval trace."""
result_rows = results or []
selection_count = len(selection) + len(assistant_selection)
payload: dict[str, Any] = {
"retrieval_trace_id": str(uuid4()),
"trigger": "tool",
"query": query,
"selection": selection,
"assistant_selection": assistant_selection,
# ``knowledge_base_count`` is retained for older trace consumers. It
# historically meant the requested selection count, not an
# authorization result, so new consumers must use the explicit fields.
"knowledge_base_count": selection_count,
"selection_count": selection_count,
"authorized_space_count": max(0, int(authorized_space_count)),
"mode": mode,
"results": result_rows,
"total_result_count": len(result_rows),
"results_truncated": False,
"partial": partial,
"warnings": list(dict.fromkeys(warnings or [])),
"fallback_knowledge_base_ids": fallback_knowledge_base_ids or [],
"knowledge_base_profiles": knowledge_base_profiles or [],
"query_plan": query_plan or {},
"statistics": statistics,
"references": references or [],
"diagnostics": diagnostics or {},
"timings_ms": {"total": round((perf_counter() - started_at) * 1000, 2)},
"zero_result_reason": None if result_rows else zero_result_reason or "KEYWORD_NO_MATCH",
}
if error:
payload["error"] = error
return json.dumps(payload, ensure_ascii=False)
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
@@ -50,6 +109,15 @@ def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str]
return list(dict.fromkeys(str(item).strip() for item in raw if str(item).strip()))[:100]
def selected_assistant_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
if not context or "assistant_knowledge_base_ids" not in context:
return None
raw = context.get("assistant_knowledge_base_ids")
if not isinstance(raw, list):
return []
return list(dict.fromkeys(str(item).strip() for item in raw if str(item).strip()))[:100]
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
agent_id = str((context or {}).get("agent_id") or "").strip()
if not agent_id or agent_id in {"default", "lead_agent"}:
@@ -76,59 +144,210 @@ async def llmwiki_search_tool(
query: Focused natural-language retrieval query.
"""
started_at = perf_counter()
local_config = get_app_config().llmwiki.local_wiki_index
resolved = get_resolved_llmwiki_runtime()
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
store = get_llmwiki_store()
user_id = get_effective_user_id()
is_admin = getattr(get_current_user(), "system_role", None) == "admin"
context = runtime.context if runtime.context else {}
selected_ids = selected_knowledge_base_ids(context)
assistant_selected_ids = selected_assistant_knowledge_base_ids(context)
allowed_ids = _agent_allowed_ids(context)
if selected_ids is None:
return "No LLMWiki knowledge base is selected for this conversation."
elif allowed_ids is not None and not set(selected_ids).issubset(set(allowed_ids)):
return "The conversation requested knowledge outside this agent's configured LLMWiki scope."
if selected_ids == []:
return "No LLMWiki knowledge base is selected for this conversation."
rows = []
for mapping_id in selected_ids:
row = await store.get_authorized(
mapping_id,
user_id,
write=False,
is_admin=is_admin,
selected_ids = selected_ids or []
assistant_selected_ids = assistant_selected_ids or []
if not selected_ids and not assistant_selected_ids:
return _trace_json(
started_at=started_at,
query=query,
selection=[],
assistant_selection=[],
zero_result_reason="NO_SELECTED_SPACE",
error="No knowledge space is selected for this conversation.",
)
if row is None:
return f"Knowledge base {mapping_id!r} is not available to the current user."
rows.append(row)
if not rows:
return "No authorized LLMWiki knowledge bases are available."
try:
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
except (RuntimeError, WikiIndexError) as exc:
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
mapping_by_id = {str(row["id"]): row for row in rows}
results = []
for item in payload["results"]:
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
results.append(visible)
return json.dumps(
{
"query": query,
"selection": [row["id"] for row in rows],
"knowledge_base_count": len(rows),
"mode": payload.get("mode") or "wiki_api",
"results": results,
"partial": payload.get("partial", False),
"warnings": payload.get("warnings", []),
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
},
ensure_ascii=False,
warnings: list[str] = []
partial = False
rows: list[dict[str, Any]] = []
authorized_assistant_ids: list[str] = []
results: list[dict[str, Any]] = []
payload: dict[str, Any] = {"mode": "empty", "partial": False, "warnings": []}
ordinary_candidate_ids = selected_ids
if allowed_ids is not None:
allowed_set = set(allowed_ids)
denied_ids = [mapping_id for mapping_id in selected_ids if mapping_id not in allowed_set]
if denied_ids:
partial = True
warnings.append("KNOWLEDGE_SCOPE_DENIED")
ordinary_candidate_ids = [mapping_id for mapping_id in selected_ids if mapping_id in allowed_set]
if ordinary_candidate_ids:
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
partial = True
warnings.append("RETRIEVAL_PROVIDER_UNAVAILABLE")
else:
try:
store = get_llmwiki_store()
for mapping_id in ordinary_candidate_ids:
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None:
partial = True
warnings.append("SPACE_NOT_FOUND_OR_UNAUTHORIZED")
continue
rows.append(row)
except Exception: # noqa: BLE001 - the assistant channel may still be usable
partial = True
warnings.append("RETRIEVAL_PROVIDER_UNAVAILABLE")
rows = []
if rows:
try:
retrieval_service = get_local_wiki_retrieval_service()
try:
index_store = get_local_wiki_index_store()
except RuntimeError:
index_store = None
if index_store is None:
try:
payload = await retrieval_service.search(query.strip(), rows, top_k_pages=20)
except TypeError as exc:
if "top_k_pages" not in str(exc):
raise
payload = await retrieval_service.search(query.strip(), rows)
payload["knowledge_base_profiles"] = [
{
"id": row.get("id"),
"name": row.get("name"),
"description": row.get("description") or "",
}
for row in rows
]
else:
payload = await advanced_search(
query=query.strip(),
mappings=rows,
retrieval_service=retrieval_service,
index_store=index_store,
top_k=20,
)
except Exception as exc: # noqa: BLE001 - preserve a successful assistant channel
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
partial = True
warnings.append(code)
payload = {"mode": "empty", "results": [], "partial": True, "warnings": [code]}
mapping_by_id = {str(row["id"]): row for row in rows}
for item in payload.get("results") or []:
if not isinstance(item, dict):
continue
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
results.append(visible)
if assistant_selected_ids:
from deerflow.persistence.assistant_knowledge import get_assistant_knowledge_store
assistant_store = get_assistant_knowledge_store()
if assistant_store is None:
partial = True
warnings.append("ASSISTANT_RETRIEVAL_PROVIDER_UNAVAILABLE")
assistant_results: list[dict[str, Any]] = []
else:
get_base = getattr(assistant_store, "get_base", None)
if not callable(get_base):
partial = True
warnings.append("ASSISTANT_RETRIEVAL_PROVIDER_UNAVAILABLE")
else:
for base_id in assistant_selected_ids:
try:
base = await get_base(base_id)
except Exception: # noqa: BLE001 - keep other spaces/channels searchable
partial = True
warnings.append("ASSISTANT_RETRIEVAL_FAILED")
continue
if base is None:
partial = True
warnings.append("ASSISTANT_SPACE_NOT_FOUND_OR_UNAUTHORIZED")
continue
authorized_assistant_ids.append(base_id)
assistant_results = []
if authorized_assistant_ids:
try:
assistant_results = await assistant_store.search_pages(
base_ids=authorized_assistant_ids,
query=query.strip(),
limit=8,
include_chunks=True,
embedding_client=get_local_wiki_search_service().embedding,
)
except Exception: # noqa: BLE001 - preserve a successful ordinary channel
partial = True
warnings.append("ASSISTANT_RETRIEVAL_FAILED")
assistant_results = []
for item in assistant_results:
results.append(
{
**item,
"knowledge_base_id": item.get("base_id"),
"knowledge_base_name": "助手知识库",
"type": "assistant_knowledge",
"source": "助手知识库",
"retrieval_mode": item.get("retrieval_mode") or "hybrid",
}
)
payload_warnings = [str(item) for item in payload.get("warnings") or []]
warnings.extend(payload_warnings)
partial = partial or bool(payload.get("partial", False))
raw_diagnostics = payload.get("diagnostics")
if isinstance(raw_diagnostics, Mapping):
diagnostics = dict(raw_diagnostics)
fallback_reason = raw_diagnostics.get("fallback_reason")
elif isinstance(raw_diagnostics, list):
diagnostics = {"ordinary_searches": raw_diagnostics}
fallback_reason = None
else:
diagnostics = {}
fallback_reason = None
diagnostics["assistant_candidates"] = sum(1 for item in results if item.get("type") == "assistant_knowledge")
diagnostics["ordinary_authorized_space_count"] = len(rows)
diagnostics["assistant_authorized_space_count"] = len(authorized_assistant_ids)
ordinary_results = [item for item in results if item.get("type") != "assistant_knowledge"]
assistant_results = [item for item in results if item.get("type") == "assistant_knowledge"]
if ordinary_results and assistant_results:
mode = "mixed"
elif ordinary_results:
mode = str(payload.get("mode") or "hybrid")
elif assistant_results:
assistant_modes = {str(item.get("retrieval_mode") or "hybrid") for item in assistant_results}
mode = next(iter(assistant_modes)) if len(assistant_modes) == 1 else "hybrid"
else:
mode = str(payload.get("mode") or "empty")
if results:
zero_result_reason = None
elif fallback_reason:
zero_result_reason = str(fallback_reason)
elif warnings:
zero_result_reason = warnings[0] if all(item in {"KNOWLEDGE_SCOPE_DENIED", "SPACE_NOT_FOUND_OR_UNAUTHORIZED", "ASSISTANT_SPACE_NOT_FOUND_OR_UNAUTHORIZED"} for item in warnings) else "RETRIEVAL_PROVIDER_UNAVAILABLE"
else:
zero_result_reason = "KEYWORD_NO_MATCH"
return _trace_json(
started_at=started_at,
query=query,
selection=selected_ids,
assistant_selection=assistant_selected_ids,
authorized_space_count=len(rows) + len(authorized_assistant_ids),
mode=mode,
results=results,
partial=partial,
warnings=warnings,
fallback_knowledge_base_ids=[str(item) for item in payload.get("fallback_knowledge_base_ids", [])],
knowledge_base_profiles=list(payload.get("knowledge_base_profiles") or []),
query_plan=dict(payload.get("query_plan") or {}),
statistics=payload.get("statistics"),
references=list(payload.get("references") or []),
diagnostics=diagnostics,
zero_result_reason=zero_result_reason,
error=("Knowledge retrieval could not complete for any selected space." if warnings and not results else None),
)
@@ -26,6 +26,104 @@ _MAX_HTML_BYTES = 4 * 1024 * 1024
_SAFE_OUTPUT_NAME = re.compile(r'^[^<>:"/\\|?*\x00-\x1f]+\.html$', re.IGNORECASE)
_FONT_SIZE_RE = re.compile(r"font-size\s*:\s*([0-9]+(?:\.[0-9]+)?)px", re.IGNORECASE)
_MIN_TEXT_FONT_SIZE = 16.0
_FLOW_FEATURES = {"flow", "flowchart", "process-flow", "process_flow", "mermaid", "流程图"}
_CHART_FEATURES = {"line", "bar", "pie", "radar", "scatter", "gantt", "chart", "echarts", "图表"}
_MAP_FEATURES = {"map", "leaflet", "geojson", "geo-json", "地图"}
_GEO_MAP_CATALOG = frozenset(
{"china-provinces", "us-states", "japan-prefectures", "taiwan-counties"}
)
def _required_visualizations(features: Iterable[str], material_brief: str = "") -> set[str]:
"""Normalize explicit features and textual source hints into runtime contracts."""
normalized = {str(value).strip().lower() for value in features if str(value).strip()}
brief = (material_brief or "").lower()
required: set[str] = set()
if normalized & _FLOW_FEATURES or re.search(r"流程图|flowchart|process[-_\s]?flow|```mermaid", brief):
required.add("flowchart")
if normalized & _CHART_FEATURES or re.search(r"图表|折线图|柱状图|饼图|雷达图|散点图|甘特图|echarts", brief):
required.add("chart")
if normalized & _MAP_FEATURES or re.search(r"地图|leaflet|经纬度|坐标点", brief):
required.add("map")
return required
def _visualization_contract(required: set[str]) -> dict[str, object]:
contracts: dict[str, dict[str, object]] = {
"flowchart": {
"renderer": "AntV G6",
"required_assets": ["g6.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-flow\"", "data-df-flow"],
"forbidden": ["Mermaid in HTML", "handwritten HTML/CSS flow nodes", "new G6.Graph"],
},
"chart": {
"renderer": "ECharts",
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-chart\"", "data-df-chart"],
},
"map": {
"renderer": "ECharts GeoJSON(Canvas)",
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
"supported_maps": sorted(_GEO_MAP_CATALOG),
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
},
}
return {name: contracts[name] for name in sorted(required)}
def _requested_page_count(preferred_page_count: int | None) -> int:
"""Default every HTML artifact to one page unless multi-page is explicit."""
if preferred_page_count is None:
return 1
return max(1, int(preferred_page_count))
def _requested_layout_mode(page_brief: str) -> str:
"""Use a fixed-screen template only when the user explicitly asks for one."""
brief = (page_brief or "").lower()
if re.search(r"大屏|驾驶舱|单屏|固定画布|1920\s*[x×*]\s*1080|big[-\s]?screen", brief):
return "screen"
return "document"
def _choose_page_profile(
*,
theme: str,
content_density: str,
required_features: list[str],
requested_page_count: int,
selection_key: str,
exclude_template_ids: list[str],
template_id: str | None,
layout_mode: str = "document",
):
multi_page = requested_page_count > 1
if not multi_page and layout_mode == "document":
normalized_theme = (theme or "").strip().lower()
document_id = "dark-document" if normalized_theme in {"dark", "deep-blue", "dark-blue"} else "light-document"
return get_template_profile(document_id)
explicit = get_template_profile(template_id) if (template_id or "").strip() else None
if explicit is not None and (explicit.page_count > 1) == multi_page:
return explicit
wrong_mode_ids = {
profile.id
for profile in list_template_profiles()
if (profile.page_count > 1) != multi_page
}
excluded = sorted(
wrong_mode_ids
| {value.strip().lower() for value in exclude_template_ids if value.strip()}
)
return choose_template(
theme=theme,
content_density=content_density,
required_features=required_features,
preferred_page_count=requested_page_count,
selection_key=selection_key,
exclude_template_ids=excluded,
)
def _thread_id(runtime: ToolRuntime[ContextT, ThreadState]) -> str:
@@ -106,6 +204,21 @@ def _edit_contract(template_id: str) -> dict[str, object]:
}
def _apply_document_theme(html: str, theme: str) -> str:
"""Set the shared responsive document template's requested color theme."""
if theme not in {"light", "dark"}:
return html
if re.search(r"<html\b[^>]*\bdata-theme=", html, flags=re.IGNORECASE):
return re.sub(
r"(<html\b[^>]*\bdata-theme\s*=\s*)[\"'][^\"']*[\"']",
rf'\1"{theme}"',
html,
count=1,
flags=re.IGNORECASE,
)
return re.sub(r"<html\b", f'<html data-theme="{theme}"', html, count=1, flags=re.IGNORECASE)
def _prepare(
runtime: ToolRuntime[ContextT, ThreadState],
*,
@@ -119,37 +232,54 @@ def _prepare(
output_name: str,
page_brief: str,
) -> str:
profile = (
get_template_profile(template_id)
if (template_id or "").strip()
else choose_template(
theme=theme,
content_density=content_density,
required_features=required_features,
preferred_page_count=preferred_page_count,
selection_key=selection_key,
exclude_template_ids=exclude_template_ids,
)
required_visualizations = _required_visualizations(required_features, page_brief)
requested_page_count = _requested_page_count(preferred_page_count)
layout_mode = _requested_layout_mode(page_brief)
profile = _choose_page_profile(
theme=theme,
content_density=content_density,
required_features=required_features,
requested_page_count=requested_page_count,
selection_key=selection_key,
exclude_template_ids=exclude_template_ids,
template_id=template_id,
layout_mode=layout_mode,
)
source = resolve_template_file(profile.id)
thread_id, user_id, workspace, outputs = _thread_directories(runtime)
filename = _safe_output_filename(output_name)
working_copy = workspace / f"{Path(filename).stem}--{profile.id}.html"
# Keep the editable copy in outputs from the beginning. The frontend live
# preview intentionally watches output writes, while workspace files are
# treated as hidden helpers. This makes the sandbox open while HTML is
# being generated instead of only after the final present_files call.
working_copy = outputs / filename
brief_path = workspace / f"{Path(filename).stem}--page-brief.json"
# Copy first, then normalize only the editable thread-local copy. This
# preserves the bundled assets exactly as supplied while giving every run a
# usable 16px-or-larger starting point.
working_copy.write_text(_apply_minimum_font_size(source.read_text(encoding="utf-8")), encoding="utf-8")
template_html = _apply_minimum_font_size(source.read_text(encoding="utf-8"))
template_html = _apply_document_theme(template_html, profile.theme)
working_copy.write_text(template_html, encoding="utf-8")
brief = {
"template": profile.to_dict(),
"requested_theme": theme or "auto",
"requested_content_density": content_density or "balanced",
"requested_page_count": requested_page_count,
"page_mode": "multi" if requested_page_count > 1 else "single",
"layout_mode": layout_mode,
"required_features": required_features,
"recommended_final_output": _virtual_path(outputs / filename, thread_id=thread_id, user_id=user_id),
"recommended_final_output": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
"working_copy": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
"user_material_brief": (page_brief or "").strip(),
"required_visualizations": sorted(required_visualizations),
"visualization_contract": _visualization_contract(required_visualizations),
"edit_contract": _edit_contract(profile.id),
"font_rule": "所有可见文字最小字号为 16px;生成后必须通过 validate 校验。",
"layout_rule": (
"普通单页报告必须是可纵向滚动的响应式文档;禁止固定 1920×1080 舞台、body overflow:hidden 和视口缩放脚本。"
if layout_mode == "document"
else "用户明确要求大屏,可使用固定单屏布局,但仍需保证各区块不重叠。"
),
}
brief_path.write_text(json.dumps(brief, ensure_ascii=False, indent=2), encoding="utf-8")
return json.dumps(
@@ -159,7 +289,11 @@ def _prepare(
"working_copy": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
"brief_path": _virtual_path(brief_path, thread_id=thread_id, user_id=user_id),
"final_output": brief["recommended_final_output"],
"next_step": "读取 brief_path 和 working_copy,改完后写入 final_output,再调用 action=validate。",
"next_step": (
"读取 brief_path 和 working_copy,严格实现 visualization_contract,"
"直接编辑 working_copy(它同时也是 final_output,可实时打开沙箱预览),"
"完成后调用 action=validate;不要再用 cp/bash 复制文件。"
),
},
ensure_ascii=False,
)
@@ -187,10 +321,223 @@ def _count_textual_content(html: str) -> int:
return len(re.sub(r"\s+", "", plain))
def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -> dict[str, object]:
def _brief_visualizations(actual: Path, workspace: Path) -> set[str]:
"""Recover prepare-time visualization requirements for final validation."""
stems = [actual.stem]
if "--" in actual.stem:
stems.append(actual.stem.split("--", 1)[0])
for stem in dict.fromkeys(stems):
brief_path = workspace / f"{stem}--page-brief.json"
if not brief_path.is_file():
continue
try:
brief = json.loads(brief_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
declared = brief.get("required_visualizations", [])
features = brief.get("required_features", [])
material = brief.get("user_material_brief", "")
values = [*(declared if isinstance(declared, list) else []), *(features if isinstance(features, list) else [])]
return _required_visualizations(values, material if isinstance(material, str) else "")
return set()
def _brief_page_count(actual: Path, workspace: Path) -> int | None:
stems = [actual.stem]
if "--" in actual.stem:
stems.append(actual.stem.split("--", 1)[0])
for stem in dict.fromkeys(stems):
brief_path = workspace / f"{stem}--page-brief.json"
if not brief_path.is_file():
continue
try:
brief = json.loads(brief_path.read_text(encoding="utf-8"))
value = int(brief.get("requested_page_count", 1))
except (OSError, ValueError, TypeError, json.JSONDecodeError):
continue
return max(1, value)
return None
def _brief_layout_mode(actual: Path, workspace: Path) -> str | None:
stems = [actual.stem]
if "--" in actual.stem:
stems.append(actual.stem.split("--", 1)[0])
for stem in dict.fromkeys(stems):
brief_path = workspace / f"{stem}--page-brief.json"
if not brief_path.is_file():
continue
try:
brief = json.loads(brief_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
value = str(brief.get("layout_mode", "")).strip().lower()
if value in {"document", "screen"}:
return value
return None
def _count_page_views(html: str) -> int:
patterns = (
r"\bdata-page\s*=",
r"\brole\s*=\s*[\"']tabpanel[\"']",
r"class\s*=\s*[\"'][^\"']*\breport-page\b[^\"']*[\"']",
r"class\s*=\s*[\"'][^\"']*\bview\b[^\"']*[\"']",
)
return max(1, *(len(re.findall(pattern, html, flags=re.IGNORECASE)) for pattern in patterns))
def _append_missing_contract_errors(errors: list[str], lowered: str, kind: str) -> None:
requirements = {
"flowchart": (
("g6.min.js", "G6 浏览器运行库 g6.min.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-flow", "data-df-flow JSON 数据块"),
("df-flow", "df-flow 流程图容器"),
),
"chart": (
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-chart", "data-df-chart JSON 数据块"),
("df-chart", "df-chart 图表容器"),
),
"map": (
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
("data-df-geo", "data-df-geo JSON 数据块"),
("df-geo-map", "df-geo-map 地图容器"),
),
}
labels = {"flowchart": "流程图", "chart": "图表", "map": "地图"}
missing = [description for marker, description in requirements[kind] if marker not in lowered]
if missing:
errors.append(f"{labels[kind]}未使用平台绘图契约,缺少:{'、'.join(missing)}。")
def _flow_layout_errors(html: str) -> list[str]:
"""Catch undersized long G6 diagrams before they overlap or become tiny."""
errors: list[str] = []
pattern = re.compile(
r"<script\b[^>]*data-df-flow\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
flags=re.IGNORECASE,
)
for flow_id, payload in pattern.findall(html):
try:
data = json.loads(payload)
except json.JSONDecodeError:
errors.append(f"流程图 {flow_id} 的 data-df-flow 不是严格 JSON。")
continue
nodes = data.get("nodes", []) if isinstance(data, dict) else []
node_count = len(nodes) if isinstance(nodes, list) else 0
tag = re.search(
rf"<[^>]+\bid\s*=\s*[\"']{re.escape(flow_id)}[\"'][^>]*>",
html,
flags=re.IGNORECASE,
)
height = None
if tag:
height_match = re.search(r"height\s*:\s*([0-9]+(?:\.[0-9]+)?)px", tag.group(0), flags=re.IGNORECASE)
if height_match:
height = float(height_match.group(1))
if height is None:
css_height = re.search(
rf"#{re.escape(flow_id)}\s*\{{[^}}]*height\s*:\s*([0-9]+(?:\.[0-9]+)?)px",
html,
flags=re.IGNORECASE,
)
if css_height:
height = float(css_height.group(1))
if height is None:
errors.append(f"流程图 {flow_id} 必须设置明确的像素高度。")
elif height < 360:
errors.append(f"流程图 {flow_id} 高度仅 {height:g}px;至少需要 360px。")
if node_count > 8:
direction = str(data.get("direction", "LR")).upper() if isinstance(data, dict) else "LR"
if direction != "TB":
errors.append(f"流程图 {flow_id} 有 {node_count} 个节点;请拆分,或改用 TB 纵向布局。")
if height is not None and height < 600:
errors.append(f"流程图 {flow_id} 有 {node_count} 个节点;TB 布局高度至少需要 600px。")
return errors
def _geo_map_errors(html: str) -> list[str]:
"""Validate the packaged GeoJSON map contract without loading a browser."""
errors: list[str] = []
pattern = re.compile(
r"<script\b[^>]*data-df-geo\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
flags=re.IGNORECASE,
)
for map_id, payload in pattern.findall(html):
try:
data = json.loads(payload)
except json.JSONDecodeError:
errors.append(f"地图 {map_id} 的 data-df-geo 不是严格 JSON。")
continue
if not isinstance(data, dict):
errors.append(f"地图 {map_id} 的 data-df-geo 必须是 JSON 对象。")
continue
if data.get("map") not in _GEO_MAP_CATALOG:
errors.append(
f"地图 {map_id} 必须使用内置地图之一:{', '.join(sorted(_GEO_MAP_CATALOG))};"
"不要引用外部地图、Leaflet 或地图瓦片。"
)
for key in ("regions", "markers", "lines"):
if key in data and not isinstance(data[key], list):
errors.append(f"地图 {map_id} 的 {key} 必须是数组。")
return errors
def _contains_chart_callback(value: object) -> bool:
"""Reject callback-shaped values: declarative chart payloads must stay JSON-only."""
if isinstance(value, str):
return bool(re.search(r"(?:\bfunction\s*\(|=>)", value))
if isinstance(value, dict):
return any(_contains_chart_callback(item) for item in value.values())
if isinstance(value, list):
return any(_contains_chart_callback(item) for item in value)
return False
def _chart_errors(html: str) -> list[str]:
"""Validate declarative ECharts payloads before the browser receives them."""
errors: list[str] = []
pattern = re.compile(
r"<script\b[^>]*data-df-chart\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
flags=re.IGNORECASE,
)
for chart_id, payload in pattern.findall(html):
try:
data = json.loads(payload)
except json.JSONDecodeError:
errors.append(
f"图表 {chart_id} 的 data-df-chart 不是严格 JSON;"
"不要写 function/回调或其他 JavaScript 表达式。"
)
continue
if not isinstance(data, dict):
errors.append(f"图表 {chart_id} 的 data-df-chart 必须是 JSON 对象。")
continue
if _contains_chart_callback(data):
errors.append(
f"图表 {chart_id} 的 data-df-chart 包含函数或箭头函数;"
"请改用字面量颜色、数据项 itemStyle 或字符串 formatter。"
)
return errors
def _quality_report(
html: str,
*,
data_mode: str,
expected_features: set[str],
expected_page_count: int | None = None,
expected_layout_mode: str | None = None,
) -> dict[str, object]:
lowered = html.lower()
errors: list[str] = []
warnings: list[str] = []
required_visualizations = _required_visualizations(expected_features)
page_views = _count_page_views(html)
if "<!doctype html" not in lowered or "</html>" not in lowered:
errors.append("HTML 不完整:需要 <!doctype html> 和 </html>。")
if "<body" not in lowered or "</body>" not in lowered:
@@ -204,6 +551,32 @@ def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -
marker = chart_markers.get(feature)
if marker and marker not in lowered:
warnings.append(f"未检测到请求的 {feature} 图表结构,请确认是否应删去该要求或补充图表。")
for kind in sorted(required_visualizations):
_append_missing_contract_errors(errors, lowered, kind)
if "flowchart" in required_visualizations:
if re.search(r"class=[\"'][^\"']*\bmermaid\b", lowered) or "mermaid.min.js" in lowered:
errors.append("HTML 流程图禁止使用 Mermaid;请转换为 data-df-flow 并由平台 G6 运行时渲染。")
if re.search(r"new\s+(?:window\.)?g6\.graph\s*\(", lowered):
errors.append("不要在 HTML 中自行初始化 G6.Graph;请只提供 data-df-flow JSON 数据块。")
errors.extend(_flow_layout_errors(html))
if "chart" in required_visualizations:
errors.extend(_chart_errors(html))
if "map" in required_visualizations:
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
errors.extend(_geo_map_errors(html))
if expected_layout_mode == "document":
if re.search(r"(?:html\s*,\s*body|body|html)\s*\{[^}]*\boverflow\s*:\s*hidden", lowered):
errors.append("普通单页报告必须允许纵向滚动,禁止对 html/body 设置 overflow:hidden。")
fixed_canvas = "1920px" in lowered and "1080px" in lowered
viewport_scaler = bool(
re.search(r"inner(?:width|height)\s*/\s*(?:1920|1080)", lowered)
or re.search(r"scale\s*\([^)]*(?:1920|1080)", lowered)
)
if fixed_canvas or viewport_scaler:
errors.append("普通单页报告禁止固定 1920×1080 舞台和整页等比缩放;请改为响应式文档布局。")
font_sizes = [float(value) for value in _FONT_SIZE_RE.findall(html)]
too_small = sorted({value for value in font_sizes if value < 16})
if too_small:
@@ -214,6 +587,10 @@ def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -
errors.append("示意图表没有标注“示意数据 / 待补充数据”,可能被误认为事实。")
if "reportdata" in lowered and "const reportdata" not in lowered:
warnings.append("检测到 reportData 引用但未检测到 const reportData 定义,请检查图表脚本。")
if expected_page_count == 1 and page_views > 1:
errors.append(f"用户未明确要求多页面,HTML 必须保持单页面;当前检测到 {page_views} 个页面视图。")
if expected_page_count is not None and expected_page_count > 1 and page_views < 2:
errors.append("用户明确要求多页面,但 HTML 未检测到多个页面视图。")
return {
"ok": not errors,
"errors": errors,
@@ -221,8 +598,10 @@ def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -
"metrics": {
"bytes": len(html.encode("utf-8")),
"text_characters": _count_textual_content(html),
"page_sections": len(re.findall(r"class=[\"'][^\"']*(?:report-page|page)[^\"']*[\"']", html, flags=re.IGNORECASE)),
"page_sections": page_views,
"has_script": "<script" in lowered,
"required_visualizations": sorted(required_visualizations),
"layout_mode": expected_layout_mode,
},
}
@@ -233,15 +612,28 @@ def _validate(
file_path: str,
data_mode: str,
expected_features: list[str],
preferred_page_count: int | None,
page_brief: str,
) -> str:
actual, thread_id, user_id = _resolve_current_thread_file(runtime, file_path)
if actual.stat().st_size > _MAX_HTML_BYTES:
raise ValueError(f"HTML 超过 {_MAX_HTML_BYTES // 1024 // 1024}MB,拒绝读取。")
html = actual.read_text(encoding="utf-8")
workspace = get_paths().sandbox_work_dir(thread_id, user_id=user_id)
persisted_visualizations = _brief_visualizations(actual, workspace)
expected_page_count = _brief_page_count(actual, workspace)
if expected_page_count is None:
expected_page_count = _requested_page_count(preferred_page_count)
expected_layout_mode = _brief_layout_mode(actual, workspace) or _requested_layout_mode(page_brief)
merged_features = {item.strip().lower() for item in expected_features if item.strip()}
merged_features.update(persisted_visualizations)
merged_features.update(_required_visualizations([], page_brief))
report = _quality_report(
html,
data_mode=data_mode,
expected_features={item.strip().lower() for item in expected_features if item.strip()},
expected_features=merged_features,
expected_page_count=expected_page_count,
expected_layout_mode=expected_layout_mode,
)
report_path = actual.with_name(f"{actual.stem}.quality.json")
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
@@ -268,26 +660,41 @@ def page_report_template_tool(
) -> str:
"""Select, copy, and quality-check reusable HTML report-page templates.
Only use this tool when the current user request explicitly asks for an HTML
file, webpage, page, dashboard, or large-screen view. A request to draw a
flowchart, chart, map, or visualization alone is not permission to use it.
HTML is generated from scratch by default: write it directly under
``/mnt/user-data/outputs/`` and use only ``validate``. Use ``catalog`` and
``prepare`` only when the user explicitly requests a template. Always
present the completed artifact; never substitute an inline HTML code block.
Use this tool instead of editing a source template in place.
Args:
action: One of ``catalog``, ``prepare``, or ``validate``. ``catalog`` lists
profiles; ``prepare`` selects/copies a source into this thread's
workspace and writes an editing brief; ``validate`` checks an edited
HTML file and writes a JSON quality report next to it.
profiles; ``prepare`` is opt-in and selects/copies a user-requested
template into this thread's outputs; ``validate`` also supports
from-scratch HTML and writes a JSON quality report next to it.
theme: ``dark`` or ``light``. Use the UI-provided presentation theme when
available; ``auto`` lets the selector use all profiles.
content_density: ``text-heavy``, ``balanced``, or ``data-heavy``.
required_features: Needed components such as ``line``, ``pie``,
``radar``, ``scatter``, ``gantt``, ``table``, ``text-panels``.
required_features: Needed components such as ``flowchart``, ``chart``,
``map``, ``line``, ``pie``, ``radar``, ``scatter``, ``gantt``,
``table``, or ``text-panels``. Preserve source Markdown Mermaid as
``flowchart`` for both ``prepare`` and ``validate``.
preferred_page_count: Preferred number of report pages. The agent may
remove unused skeleton pages after preparation.
use a value greater than one only when the current user explicitly
asks for multiple pages or specifies a page count. Omitted defaults
to one page; pass it to validate even when prepare was skipped.
selection_key: Stable request/thread-derived text used to vary equally
good template choices.
exclude_template_ids: Recently-used template ids to avoid where possible.
template_id: Optional explicit catalog id. Omit to select automatically.
output_name: Final safe HTML filename, e.g. ``经营月报.html``.
page_brief: Concise summary of supplied materials and desired output.
Mention every source flowchart, chart, and map; prepare persists
these requirements, while direct validation reads them from this
argument. Always pass it to validate when prepare was skipped.
file_path: For ``validate`` only: current-thread HTML under workspace or
outputs.
data_mode: ``provided`` for factual data, ``illustrative`` when charts
@@ -315,6 +722,8 @@ def page_report_template_tool(
file_path=file_path,
data_mode=data_mode,
expected_features=required_features or [],
preferred_page_count=preferred_page_count,
page_brief=page_brief,
)
raise ValueError("action 仅支持 catalog、prepare、validate。")
@@ -4,19 +4,77 @@ from __future__ import annotations
import json
import logging
from pathlib import PurePosixPath
from typing import TYPE_CHECKING
from langchain.tools import ToolRuntime, tool
if TYPE_CHECKING:
from deerflow.skills.storage import SkillStorage
from deerflow.skills.types import Skill
logger = logging.getLogger(__name__)
_ROUNDTABLE_COORDINATOR_AGENT_ID = "roundtable-coordinator"
COORDINATOR_AGENT_IDS = frozenset(
{_ROUNDTABLE_COORDINATOR_AGENT_ID, "task-analysis-coordinator"}
)
COORDINATOR_ONLY_SKILL_NAMES = frozenset({"agent_orchestration", "agent-orchestration"})
class SkillPathUnavailableError(PermissionError):
"""Raised when a model tool targets an unknown or offline skill path."""
def require_enabled_skill_path(
path: str,
*,
container_path: str = "/mnt/skills",
storage: SkillStorage | None = None,
) -> Skill:
"""Return the discovered enabled skill containing ``path`` or deny access.
Matching uses the discovered directory instead of assuming it equals the
frontmatter name. Root/category-wide paths and unregistered files fail
closed; models discover skills through ``skill_list``/``search_skills``.
"""
normalized_container = container_path.replace("\\", "/").rstrip("/") or "/"
normalized_path = path.replace("\\", "/").rstrip("/") or "/"
if normalized_path != normalized_container and not normalized_path.startswith(f"{normalized_container}/"):
raise SkillPathUnavailableError("Path is outside the skills mount")
if ".." in PurePosixPath(normalized_path).parts:
raise SkillPathUnavailableError("Skill path traversal is not allowed")
try:
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.skills.storage import get_or_new_skill_storage
# SkillStorage intentionally degrades open on extensions-config errors;
# model-facing file access must instead fail closed.
extensions = ExtensionsConfig.from_file()
catalog = (storage or get_or_new_skill_storage()).load_skills(enabled_only=False)
except Exception as exc:
raise SkillPathUnavailableError("Skill availability could not be verified") from exc
matches: list[tuple[int, Skill]] = []
for skill in catalog:
skill_root = skill.get_container_path(normalized_container).replace("\\", "/").rstrip("/")
if normalized_path == skill_root or normalized_path.startswith(f"{skill_root}/"):
matches.append((len(skill_root), skill))
if not matches:
raise SkillPathUnavailableError("Skill path is unavailable")
skill = max(matches, key=lambda item: item[0])[1]
category = skill.category.value if hasattr(skill.category, "value") else str(skill.category)
if not extensions.is_skill_enabled(skill.name, category):
raise SkillPathUnavailableError("Skill path is unavailable")
return skill
def can_access_coordinator_only_skill(agent_id: str | None, skill_name: str) -> bool:
"""Return whether this runtime identity may discover/read a reserved orchestration skill."""
normalized = (skill_name or "").strip().lower()
return normalized not in COORDINATOR_ONLY_SKILL_NAMES or agent_id == _ROUNDTABLE_COORDINATOR_AGENT_ID
return normalized not in COORDINATOR_ONLY_SKILL_NAMES or agent_id in COORDINATOR_AGENT_IDS
def is_coordinator_only_skill_path(path: str) -> bool:
@@ -66,22 +124,31 @@ def _load_agent_skill_whitelist(agent_id: str | None, extra_skills: list[str] |
through this tool.
- Non-empty set → only these names are visible to this agent.
Exception — the deep-research collector (``deep-research-collector``) has no
whitelist: it sees every skill like the default lead agent, matching the
runtime allowlist in ``lead_agent.agent``. Its on-disk ``skills:`` entry is
ignored.
``extra_skills`` only widens the **deep-research collector** allowlist
for one run (report-structure 检索来源). Other agents ignore it.
"""
if not agent_id:
return None
try:
from deerflow.agents.deep_research.retrieval import COLLECTOR_AGENT_ID
from deerflow.config.agents_config import load_agent_config
cfg = load_agent_config(agent_id)
if cfg is None:
return None
allowed = {
name
for name in (cfg.skills or [])
if can_access_coordinator_only_skill(agent_id, name)
}
if agent_id == COLLECTOR_AGENT_ID:
allowed = None # collector: all skills, like the default lead agent
else:
allowed = {
name
for name in (cfg.skills or [])
if can_access_coordinator_only_skill(agent_id, name)
}
except Exception:
logger.debug("Failed to load agent config for skill_list filter", exc_info=True)
return None
@@ -89,8 +156,8 @@ def _load_agent_skill_whitelist(agent_id: str | None, extra_skills: list[str] |
if extra_skills:
from deerflow.agents.deep_research.retrieval import merge_collector_skill_allowlist
merged = merge_collector_skill_allowlist(agent_id, list(allowed), extra_skills)
return set(merged or [])
merged = merge_collector_skill_allowlist(agent_id, None if allowed is None else list(allowed), extra_skills)
return set(merged or []) if merged is not None else None
return allowed
@@ -169,20 +236,13 @@ def skill_view_impl(name: str, agent_id: str | None, extra_skills: list[str] | N
storage = get_or_new_skill_storage()
try:
if storage.custom_skill_exists(name):
content = storage.read_custom_skill(name)
elif storage.public_skill_exists(name):
skill_file = storage.get_skills_root_path()
from pathlib import Path
from deerflow.skills.types import SKILL_MD_FILE
path = Path(skill_file) / "public" / name / SKILL_MD_FILE
if not path.exists():
return f"Skill '{name}' not found."
content = path.read_text(encoding="utf-8")
else:
# Discover through the enabled catalog instead of reconstructing a
# path from ``name``. Besides enforcing the online/offline switch,
# this preserves skills whose directory differs from frontmatter name.
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
if skill is None:
return f"Skill '{name}' not found."
content = skill.skill_file.read_text(encoding="utf-8")
except Exception as e:
logger.debug("skill_view failed for %r: %s", name, e)
return f"Error reading skill '{name}': {e}"
@@ -353,7 +413,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
try:
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=False)
skills = storage.load_skills(enabled_only=True)
usage = all_entries()
result = []
for skill in skills:
@@ -1,5 +1,6 @@
import base64
import mimetypes
import base64
import io
import mimetypes
from pathlib import Path
from typing import Annotated
@@ -17,7 +18,10 @@ _ALLOWED_IMAGE_VIRTUAL_ROOTS = (
f"{VIRTUAL_PATH_PREFIX}/outputs",
)
_ALLOWED_IMAGE_VIRTUAL_ROOTS_TEXT = ", ".join(_ALLOWED_IMAGE_VIRTUAL_ROOTS)
_MAX_IMAGE_BYTES = 20 * 1024 * 1024
_MAX_IMAGE_BYTES = 100 * 1024 * 1024
_MAX_IMAGE_PIXELS = 40_000_000
_MAX_ANALYSIS_SIDE = 2048
_MAX_ANALYSIS_BYTES = 4 * 1024 * 1024
_EXTENSION_TO_MIME = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
@@ -40,10 +44,44 @@ def _detect_image_mime(image_data: bytes) -> str | None:
return None
def _sanitize_image_error(error: Exception, thread_data: ThreadDataState | None) -> str:
def _sanitize_image_error(error: Exception, thread_data: ThreadDataState | None) -> str:
from deerflow.sandbox.tools import mask_local_paths_in_output
return mask_local_paths_in_output(f"{type(error).__name__}: {error}", thread_data)
return mask_local_paths_in_output(f"{type(error).__name__}: {error}", thread_data)
def _bounded_analysis_image(image_data: bytes, mime_type: str) -> tuple[bytes, str]:
"""Validate decoded dimensions and create a bounded vision derivative."""
try:
from PIL import Image
except ImportError as exc:
raise RuntimeError("Pillow is required for safe image dimension validation") from exc
with Image.open(io.BytesIO(image_data)) as image:
width, height = image.size
if width <= 0 or height <= 0 or width * height > _MAX_IMAGE_PIXELS:
raise ValueError(f"decoded image dimensions are unsafe: {width}x{height}")
if max(width, height) <= _MAX_ANALYSIS_SIDE and len(image_data) <= _MAX_ANALYSIS_BYTES:
image.verify()
return image_data, mime_type
with Image.open(io.BytesIO(image_data)) as image:
image.thumbnail((_MAX_ANALYSIS_SIDE, _MAX_ANALYSIS_SIDE))
output = io.BytesIO()
if mime_type == "image/jpeg":
image.convert("RGB").save(output, format="JPEG", quality=85, optimize=True)
output_mime = "image/jpeg"
else:
image.save(output, format="PNG", optimize=True)
output_mime = "image/png"
bounded = output.getvalue()
if len(bounded) > _MAX_ANALYSIS_BYTES:
output = io.BytesIO()
image.thumbnail((1536, 1536))
image.convert("RGB").save(output, format="JPEG", quality=82, optimize=True)
bounded = output.getvalue()
output_mime = "image/jpeg"
return bounded, output_mime
@tool("view_image", parse_docstring=True)
@@ -140,9 +178,16 @@ def view_image_tool(
return Command(
update={"messages": [ToolMessage(f"Error reading image metadata: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
)
if image_size > _MAX_IMAGE_BYTES:
return Command(
update={"messages": [ToolMessage(f"Error: Image file is too large: {image_size} bytes. Maximum supported size is {_MAX_IMAGE_BYTES} bytes", tool_call_id=tool_call_id)]},
try:
from deerflow.uploads.policy import get_runtime_upload_policy
image_limit = int(((get_runtime_upload_policy().get("groups") or {}).get("image") or {}).get("max_bytes") or _MAX_IMAGE_BYTES)
except Exception:
image_limit = _MAX_IMAGE_BYTES
image_limit = min(max(1, image_limit), _MAX_IMAGE_BYTES)
if image_size > image_limit:
return Command(
update={"messages": [ToolMessage(f"Error: Image file is too large: {image_size} bytes. Maximum supported size is {image_limit} bytes", tool_call_id=tool_call_id)]},
)
# Read image file and convert to base64
@@ -159,12 +204,18 @@ def view_image_tool(
return Command(
update={"messages": [ToolMessage("Error: File contents do not match a supported image format", tool_call_id=tool_call_id)]},
)
if detected_mime_type != expected_mime_type:
if detected_mime_type != expected_mime_type:
return Command(
update={"messages": [ToolMessage(f"Error: Image contents are {detected_mime_type}, but file extension indicates {expected_mime_type}", tool_call_id=tool_call_id)]},
)
mime_type = detected_mime_type
image_base64 = base64.b64encode(image_data).decode("utf-8")
mime_type = detected_mime_type
try:
image_data, mime_type = _bounded_analysis_image(image_data, mime_type)
except Exception as e:
return Command(
update={"messages": [ToolMessage(f"Error validating image dimensions: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
)
image_base64 = base64.b64encode(image_data).decode("utf-8")
# Delegated recognition: when a dedicated vision model is configured, send
# the image to that model and return its description as the tool result, so
@@ -135,8 +135,14 @@ def get_available_tools(
builtin_tools.append(scheduled_task_tool)
# Memory V2:记忆与 builtin provider 都启用时,暴露 memory 工具给 LLM 主动写入
memory_config = getattr(config, "memory", None)
if memory_config is not None and getattr(memory_config, "enabled", False):
memory_config = getattr(config, "memory", None)
try:
from deerflow.config.system_settings import get_memory_runtime_settings
memory_runtime_enabled = get_memory_runtime_settings().enabled
except Exception: # noqa: BLE001 - settings failure must keep memory off
memory_runtime_enabled = False
if memory_runtime_enabled and memory_config is not None:
builtin_config = getattr(memory_config, "builtin", None)
if builtin_config is not None and getattr(builtin_config, "enabled", False):
builtin_tools.append(memory_tool)
@@ -177,12 +183,14 @@ def get_available_tools(
builtin_tools.append(read_peer_delivery_tool)
# Conditional LLMWiki mode: the tool is absent when no effective WeKnora
# address is configured, keeping legacy deployments behavior-identical.
# Unified selected-space retrieval is available for local Wiki snapshots,
# WeKnora, or the assistant Wiki repository.
try:
from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
from deerflow.persistence.assistant_knowledge import get_assistant_knowledge_store
if get_resolved_llmwiki_runtime(config).weknora_enabled:
local_wiki = getattr(getattr(config.llmwiki, "local_wiki_index", None), "enabled", False)
if get_resolved_llmwiki_runtime(config).weknora_enabled or local_wiki or get_assistant_knowledge_store() is not None:
from deerflow.tools.builtins.llmwiki_search_tool import llmwiki_search_tool
builtin_tools.append(llmwiki_search_tool)