sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+343
-22
@@ -31,6 +31,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Annotated, Any
|
||||
|
||||
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
|
||||
@@ -174,12 +175,173 @@ _KNOWLEDGE_WRITE_INSTRUCTION = (
|
||||
"写成报告主线或用其填充多个章节。参考文献可标为基于公开领域知识,不要用「材料未提供」占整节。"
|
||||
)
|
||||
|
||||
_SOLUTION_REPORT_CITATION_INSTRUCTION = (
|
||||
"最终方案正文不得出现 [1]、[2]、【1】、【2】等数字引用标记。"
|
||||
"如需保留参考来源,请在参考来源小节中使用无编号项目符号,直接写来源名称和链接。"
|
||||
)
|
||||
|
||||
_CITATION_DIGITS_PATTERN = r"[0-90-9]{1,3}"
|
||||
_CITATION_SEQUENCE_PATTERN = (
|
||||
rf"{_CITATION_DIGITS_PATTERN}"
|
||||
rf"(?:\s*(?:[,,、;;]|[-–—~~])\s*{_CITATION_DIGITS_PATTERN})*"
|
||||
)
|
||||
_NUMERIC_CITATION_TOKEN_PATTERN = (
|
||||
rf"(?:\[\[\s*{_CITATION_SEQUENCE_PATTERN}\s*\]\]"
|
||||
rf"|[\[[【]\s*{_CITATION_SEQUENCE_PATTERN}\s*[\]]】])"
|
||||
)
|
||||
_NUMERIC_CITATION_TOKEN_RE = re.compile(_NUMERIC_CITATION_TOKEN_PATTERN)
|
||||
_NUMERIC_CITATION_LINK_RE = re.compile(
|
||||
rf"\[\s*{_CITATION_DIGITS_PATTERN}\s*\]"
|
||||
r"\((?P<target>https?://[^)\s]+|#reference-[^)]+)\)"
|
||||
)
|
||||
_REFERENCE_ENTRY_PREFIX_RE = re.compile(
|
||||
rf"^(?P<indent>\s*)(?:[-*+]\s*)?{_NUMERIC_CITATION_TOKEN_PATTERN}\s+(?=\S)"
|
||||
)
|
||||
_REFERENCE_LINK_ENTRY_PREFIX_RE = re.compile(
|
||||
rf"^(?P<indent>\s*)(?:[-*+]\s*)?"
|
||||
rf"\[\s*{_CITATION_DIGITS_PATTERN}\s*\]"
|
||||
r"\((?P<target>https?://[^)\s]+|#reference-[^)]+)\)\s*"
|
||||
)
|
||||
_REFERENCE_DEFINITION_PREFIX_RE = re.compile(
|
||||
rf"^(?P<indent>\s*)(?:[-*+]\s*)?"
|
||||
rf"{_NUMERIC_CITATION_TOKEN_PATTERN}\s*:\s*"
|
||||
)
|
||||
_REFERENCE_HEADING_RE = re.compile(
|
||||
r"^#{1,6}\s*(?:参考来源|参考文献|references|sources)\s*:?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_MARKDOWN_HEADING_RE = re.compile(r"^#{1,6}\s+\S")
|
||||
_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
|
||||
_INLINE_CODE_RE = re.compile(r"(`+[^`\n]*`+)")
|
||||
|
||||
|
||||
def _strip_numeric_citations_from_prose(value: str) -> str:
|
||||
"""Remove numeric citations while preserving inline-code examples."""
|
||||
|
||||
def _replace_numeric_link(match: re.Match[str]) -> str:
|
||||
target = match.group("target")
|
||||
return "" if target.startswith("#reference-") else f"<{target}>"
|
||||
|
||||
parts = _INLINE_CODE_RE.split(value)
|
||||
for index in range(0, len(parts), 2):
|
||||
prose = _NUMERIC_CITATION_LINK_RE.sub(_replace_numeric_link, parts[index])
|
||||
parts[index] = _NUMERIC_CITATION_TOKEN_RE.sub("", prose)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def strip_solution_numeric_citations(markdown: str) -> str:
|
||||
"""Return a clean solution report without numbered citation markers.
|
||||
|
||||
Research sources remain useful in a solution project, so entries under a
|
||||
reference heading are converted to ordinary unordered-list items instead
|
||||
of being deleted. Fenced code is left untouched because ``[1]`` may be
|
||||
valid Mermaid or source-code syntax there.
|
||||
"""
|
||||
|
||||
if not markdown:
|
||||
return markdown
|
||||
|
||||
lines: list[str] = []
|
||||
in_fence = False
|
||||
fence_char = ""
|
||||
fence_width = 0
|
||||
in_references = False
|
||||
for line in markdown.splitlines(keepends=True):
|
||||
raw_line = line.rstrip("\r\n")
|
||||
newline = line[len(raw_line) :]
|
||||
fence_match = _FENCE_RE.match(raw_line)
|
||||
if fence_match:
|
||||
marker = fence_match.group(1)
|
||||
marker_char = marker[0]
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence_char = marker_char
|
||||
fence_width = len(marker)
|
||||
elif marker_char == fence_char and len(marker) >= fence_width:
|
||||
in_fence = False
|
||||
fence_char = ""
|
||||
fence_width = 0
|
||||
lines.append(line)
|
||||
continue
|
||||
if in_fence:
|
||||
lines.append(line)
|
||||
continue
|
||||
|
||||
stripped = raw_line.strip()
|
||||
if _REFERENCE_HEADING_RE.match(stripped):
|
||||
in_references = True
|
||||
lines.append(line)
|
||||
continue
|
||||
if in_references and _MARKDOWN_HEADING_RE.match(stripped):
|
||||
in_references = False
|
||||
|
||||
cleaned = raw_line
|
||||
if in_references:
|
||||
cleaned = _REFERENCE_LINK_ENTRY_PREFIX_RE.sub(
|
||||
lambda match: (
|
||||
f"{match.group('indent')}- "
|
||||
+ (f"<{match.group('target')}> " if not match.group("target").startswith("#") else "")
|
||||
),
|
||||
cleaned,
|
||||
)
|
||||
cleaned = _REFERENCE_DEFINITION_PREFIX_RE.sub(
|
||||
lambda match: f"{match.group('indent')}- ",
|
||||
cleaned,
|
||||
)
|
||||
cleaned = _REFERENCE_ENTRY_PREFIX_RE.sub(
|
||||
lambda match: f"{match.group('indent')}- ",
|
||||
cleaned,
|
||||
)
|
||||
cleaned = _strip_numeric_citations_from_prose(cleaned)
|
||||
lines.append(cleaned + newline)
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _report_requires_unnumbered_sources(runtime_context: Mapping[str, Any]) -> bool:
|
||||
"""Return whether this run's report must avoid numbered source markers."""
|
||||
|
||||
project_context = runtime_context.get("analysis_project")
|
||||
project_mode = (
|
||||
str(project_context.get("mode") or "").strip().lower()
|
||||
if isinstance(project_context, Mapping)
|
||||
else ""
|
||||
)
|
||||
retrieval_mode = str(
|
||||
runtime_context.get("knowledge_retrieval_mode") or "classic"
|
||||
).strip().lower()
|
||||
return project_mode == "solution" or retrieval_mode == "smart_reasoning"
|
||||
|
||||
|
||||
_WORKBENCH_CONFIG_KEYS = frozenset(
|
||||
{
|
||||
"custom_outline",
|
||||
"structure_mode",
|
||||
"generate_images",
|
||||
"max_generated_images",
|
||||
"language",
|
||||
"tone",
|
||||
"report_instruction",
|
||||
"retrieval_directions",
|
||||
"retrieval_skills",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _coerce_workbench_report_config(raw: Any) -> dict[str, Any]:
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
return {key: raw[key] for key in _WORKBENCH_CONFIG_KEYS if key in raw and raw[key] is not None}
|
||||
|
||||
|
||||
def build_writing_research_config(
|
||||
*,
|
||||
focus: str | None = None,
|
||||
thinking_enabled: bool = False,
|
||||
model_name: str | None = None,
|
||||
project_mode: str | None = None,
|
||||
project_report_config: dict | None = None,
|
||||
workbench_report_config: dict | None = None,
|
||||
unnumbered_sources: bool = False,
|
||||
) -> DeepResearchConfig:
|
||||
"""组装聊天写作模式的研究配置:basic + 通用分析结构 + adaptive。
|
||||
|
||||
@@ -187,16 +349,66 @@ def build_writing_research_config(
|
||||
``model_name`` / ``thinking_enabled`` 继承自聊天 run 的模型选择——同一模型
|
||||
覆盖 fast/smart/strategic 三个角色,使报告管线与对话用同一个模型和思考
|
||||
开关(缺省回落系统默认模型)。
|
||||
|
||||
工作台确认卡传入 ``workbench_report_config`` 时沿用卡片上的大纲/结构/配图
|
||||
等选项,但 **mode 仍强制 basic**:那是普通问答写作模式在脆弱服务器上能
|
||||
写完的同一条 runner,避免再走 deep/multi_agent 的 job 路径。
|
||||
``unnumbered_sources`` 只改变引用写法,不切换方案模式或固定大纲。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
if focus and focus.strip():
|
||||
parts.append(focus.strip())
|
||||
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
|
||||
from deerflow.agents.analysis_project import SOLUTION_OUTLINE
|
||||
|
||||
fixed_solution = project_mode == "solution"
|
||||
project_report_config = project_report_config or {}
|
||||
workbench = _coerce_workbench_report_config(workbench_report_config)
|
||||
if workbench:
|
||||
outline = str(workbench.get("custom_outline") or "").strip() or GENERAL_OUTLINE
|
||||
structure_mode = workbench.get("structure_mode") or "adaptive"
|
||||
extra_instruction = str(workbench.get("report_instruction") or "").strip()
|
||||
if extra_instruction:
|
||||
parts.append(extra_instruction)
|
||||
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
|
||||
if fixed_solution or unnumbered_sources:
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
return DeepResearchConfig(
|
||||
mode="basic",
|
||||
collection_mode="legacy",
|
||||
custom_outline=outline,
|
||||
structure_mode=structure_mode if structure_mode in ("fixed", "adaptive") else "adaptive",
|
||||
retrieval_directions=list(workbench.get("retrieval_directions") or []),
|
||||
retrieval_skills=list(workbench.get("retrieval_skills") or []),
|
||||
generate_images=bool(workbench.get("generate_images")),
|
||||
max_generated_images=workbench.get("max_generated_images") or 2,
|
||||
language=str(workbench.get("language") or "zh-CN"),
|
||||
tone=workbench.get("tone") or "objective",
|
||||
thinking_enabled=thinking_enabled,
|
||||
fast_model=model_name,
|
||||
smart_model=model_name,
|
||||
strategic_model=model_name,
|
||||
report_instruction="\n".join(parts),
|
||||
).clamp()
|
||||
|
||||
outline = project_report_config.get("custom_outline") or SOLUTION_OUTLINE
|
||||
structure_mode = project_report_config.get("structure_mode", "fixed") if fixed_solution else "adaptive"
|
||||
if fixed_solution:
|
||||
parts.append(
|
||||
"遵循项目所选报告结构及固定/弹性约束。引用本项目对话资料;"
|
||||
"缺失事实、预算、责任人标注待补充,不得编造。"
|
||||
)
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
else:
|
||||
parts.append(_KNOWLEDGE_WRITE_INSTRUCTION)
|
||||
if unnumbered_sources:
|
||||
parts.append(_SOLUTION_REPORT_CITATION_INSTRUCTION)
|
||||
return DeepResearchConfig(
|
||||
mode="basic",
|
||||
collection_mode="legacy",
|
||||
custom_outline=GENERAL_OUTLINE,
|
||||
structure_mode="adaptive",
|
||||
custom_outline=outline if fixed_solution else GENERAL_OUTLINE,
|
||||
structure_mode=structure_mode,
|
||||
retrieval_directions=project_report_config.get("retrieval_directions", []) if fixed_solution else [],
|
||||
retrieval_skills=project_report_config.get("retrieval_skills", []) if fixed_solution else [],
|
||||
generate_images=False,
|
||||
thinking_enabled=thinking_enabled,
|
||||
fast_model=model_name,
|
||||
@@ -206,6 +418,54 @@ def build_writing_research_config(
|
||||
).clamp()
|
||||
|
||||
|
||||
async def _persist_workbench_session(
|
||||
*,
|
||||
session_id: str,
|
||||
status: str,
|
||||
report_markdown: str | None = None,
|
||||
source_count: int | None = None,
|
||||
error: str | None = None,
|
||||
workbench_report_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Best-effort session row update so refresh/follow-up still see the report.
|
||||
|
||||
Failures are logged only — the chat thread already has report.md in artifacts
|
||||
and the custom-event stream already showed the live draft.
|
||||
"""
|
||||
session_id = str(session_id or "").strip()
|
||||
if not session_id:
|
||||
return
|
||||
try:
|
||||
from deerflow.persistence.deep_research_sessions import get_default_store
|
||||
|
||||
store = get_default_store()
|
||||
if store is None:
|
||||
return
|
||||
fields: dict[str, Any] = {"status": status, "active_job_id": None, "error": error}
|
||||
if report_markdown is not None:
|
||||
fields["report_markdown"] = report_markdown
|
||||
fields["report_html"] = None
|
||||
if source_count is not None:
|
||||
fields["source_count"] = source_count
|
||||
if workbench_report_config:
|
||||
row = await store.get(session_id, user_id=None)
|
||||
snapshot = dict((row or {}).get("config_snapshot") or {})
|
||||
snapshot.update(_coerce_workbench_report_config(workbench_report_config))
|
||||
snapshot["mode"] = "basic"
|
||||
snapshot["collection_mode"] = "legacy"
|
||||
fields["config_snapshot"] = snapshot
|
||||
# Session id is already scoped to this confirm-card write; skip the
|
||||
# request user filter so a LangGraph tool task without contextvar
|
||||
# still lands the completed report (the file card keys off this row).
|
||||
await store.update(session_id, user_id=None, **fields)
|
||||
except Exception: # noqa: BLE001 - 落库失败不能打断写作管线
|
||||
logger.exception(
|
||||
"[deep_research_report] workbench session persist failed session=%s status=%s",
|
||||
session_id,
|
||||
status,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_chat_model(context: dict[str, Any] | None) -> tuple[str | None, bool, bool]:
|
||||
"""从 run 上下文解析聊天选择的模型与思考开关(P2:管线继承聊天模型)。
|
||||
|
||||
@@ -394,7 +654,7 @@ async def deep_research_report_tool(
|
||||
|
||||
The tool harvests the materials already produced in this conversation (search /
|
||||
knowledge / skill / file tool results) and writes a full research report
|
||||
(general-analysis structure, numbered reference section) to
|
||||
(general-analysis structure, with source provenance) to
|
||||
/mnt/user-data/outputs/report.md in this thread. The file is presented
|
||||
automatically; do NOT write the report body yourself with write_file.
|
||||
|
||||
@@ -434,20 +694,45 @@ async def deep_research_report_tool(
|
||||
query = topic.strip()
|
||||
state = getattr(runtime, "state", None) or {}
|
||||
conversation_messages = state.get("messages") or []
|
||||
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query)
|
||||
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
|
||||
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
|
||||
anchor_index = topic_anchor_index(conversation_messages, topic=query)
|
||||
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
|
||||
harvested_count = len(materials)
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
|
||||
thread_id,
|
||||
len(harvested),
|
||||
harvested_count,
|
||||
dropped_count,
|
||||
anchor_index,
|
||||
)
|
||||
runtime_context = runtime.context or {}
|
||||
project_context = runtime_context.get("analysis_project") or {}
|
||||
if project_context.get("mode") == "solution" and not runtime_context.get(
|
||||
"solution_requirements_confirmed"
|
||||
):
|
||||
return Command(
|
||||
update={
|
||||
"messages": [
|
||||
ToolMessage(
|
||||
content=("当前仍处于正常问答阶段,不能生成首份方案。请继续自然理解用户意图;信息足够后输出简短的意图理解、合法意图快照及就绪标记,并等待用户点击‘结束问答并生成方案’后再调用本工具。"),
|
||||
tool_call_id=tool_call_id,
|
||||
name="deep_research_report",
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
workbench_report_config = _coerce_workbench_report_config(runtime_context.get("writing_report_config"))
|
||||
workbench_session_id = str(runtime_context.get("workbench_session_id") or "").strip()
|
||||
force_no_materials = bool(runtime_context.get("force_no_materials"))
|
||||
harvest_options = {"include_rag_sources": True} if project_context.get("mode") == "solution" else {}
|
||||
if force_no_materials:
|
||||
harvested, dropped_count, harvested_count = [], 0, 0
|
||||
materials: list[ResearchMaterial] = []
|
||||
logger.info("[deep_research_report] thread=%s force_no_materials; writing with empty pool", thread_id)
|
||||
else:
|
||||
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query, **harvest_options)
|
||||
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
|
||||
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
|
||||
anchor_index = topic_anchor_index(conversation_messages, topic=query)
|
||||
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
|
||||
harvested_count = len(materials)
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
|
||||
thread_id,
|
||||
len(harvested),
|
||||
harvested_count,
|
||||
dropped_count,
|
||||
anchor_index,
|
||||
)
|
||||
|
||||
if harvested_count < _MIN_HARVESTED_MATERIALS:
|
||||
logger.info(
|
||||
@@ -476,7 +761,17 @@ async def deep_research_report_tool(
|
||||
focus=focus,
|
||||
thinking_enabled=thinking_enabled,
|
||||
model_name=model_name,
|
||||
project_mode=project_context.get("mode"),
|
||||
project_report_config=project_context.get("report_config"),
|
||||
workbench_report_config=workbench_report_config or None,
|
||||
unnumbered_sources=_report_requires_unnumbered_sources(runtime_context),
|
||||
)
|
||||
if workbench_session_id:
|
||||
await _persist_workbench_session(
|
||||
session_id=workbench_session_id,
|
||||
status="running",
|
||||
workbench_report_config=workbench_report_config or None,
|
||||
)
|
||||
llm_backend = DeerFlowCompletionBackend(
|
||||
research_config,
|
||||
event_sink=events,
|
||||
@@ -508,12 +803,36 @@ async def deep_research_report_tool(
|
||||
result = None
|
||||
error_message: str | None = None
|
||||
try:
|
||||
result = await DeepResearchEngine().run(request, adapters=bundle)
|
||||
generated_result = await DeepResearchEngine().run(request, adapters=bundle)
|
||||
if _report_requires_unnumbered_sources(runtime_context):
|
||||
cleaned_report = strip_solution_numeric_citations(generated_result.report_markdown)
|
||||
if cleaned_report != generated_result.report_markdown:
|
||||
await bundle.artifacts.write_text("report.md", cleaned_report)
|
||||
generated_result = generated_result.model_copy(update={"report_markdown": cleaned_report})
|
||||
result = generated_result
|
||||
except Exception as exc: # noqa: BLE001 - 工具错误以 ToolMessage 回给模型自纠
|
||||
logger.exception("[deep_research_report] thread=%s research failed", thread_id)
|
||||
error_message = f"深度研究报告生成失败:{exc}"
|
||||
|
||||
# 终态帧:无论成败都发,前端据此收尾进度条 / 停止流式预览。
|
||||
source_count = len(result.source_ids or []) if result is not None else 0
|
||||
if workbench_session_id:
|
||||
if result is None:
|
||||
await _persist_workbench_session(
|
||||
session_id=workbench_session_id,
|
||||
status="failed",
|
||||
error=error_message,
|
||||
)
|
||||
else:
|
||||
await _persist_workbench_session(
|
||||
session_id=workbench_session_id,
|
||||
status="completed",
|
||||
report_markdown=result.report_markdown or "",
|
||||
source_count=max(source_count, harvested_count),
|
||||
workbench_report_config=workbench_report_config or None,
|
||||
)
|
||||
|
||||
# 终态帧放在落库之后:前端收到 tool_finished 再 refetch 时会话已是 completed,
|
||||
# 消息流才能立刻插入 report.md 文件卡片。
|
||||
if writer is not None:
|
||||
try:
|
||||
writer(
|
||||
@@ -547,14 +866,16 @@ async def deep_research_report_tool(
|
||||
}
|
||||
)
|
||||
|
||||
source_count = len(result.source_ids or [])
|
||||
summary = f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s completed (sources=%d, report_chars=%d)",
|
||||
thread_id,
|
||||
source_count,
|
||||
len(result.report_markdown or ""),
|
||||
)
|
||||
summary = (
|
||||
f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,"
|
||||
f"已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
|
||||
)
|
||||
return Command(
|
||||
update={
|
||||
"artifacts": [REPORT_VIRTUAL_PATH],
|
||||
|
||||
+2076
File diff suppressed because it is too large.
Load diff
+340
-48
@@ -2,10 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langgraph.typing import ContextT
|
||||
@@ -13,7 +16,8 @@ from langgraph.typing import ContextT
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
|
||||
from deerflow.integrations.weknora.local_index.advanced import advanced_search
|
||||
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_index_store, get_local_wiki_retrieval_service, get_local_wiki_search_service
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
|
||||
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
|
||||
@@ -23,6 +27,62 @@ _CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
|
||||
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
|
||||
|
||||
|
||||
def _trace_json(
|
||||
*,
|
||||
started_at: float,
|
||||
query: str,
|
||||
selection: list[str],
|
||||
assistant_selection: list[str],
|
||||
authorized_space_count: int = 0,
|
||||
mode: str = "empty",
|
||||
results: list[dict[str, Any]] | None = None,
|
||||
partial: bool = False,
|
||||
warnings: list[str] | None = None,
|
||||
fallback_knowledge_base_ids: list[str] | None = None,
|
||||
knowledge_base_profiles: list[dict[str, Any]] | None = None,
|
||||
query_plan: dict[str, Any] | None = None,
|
||||
statistics: dict[str, Any] | None = None,
|
||||
references: list[dict[str, Any]] | None = None,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
zero_result_reason: str | None = None,
|
||||
error: str | None = None,
|
||||
) -> str:
|
||||
"""Serialize every tool outcome as a durable retrieval trace."""
|
||||
|
||||
result_rows = results or []
|
||||
selection_count = len(selection) + len(assistant_selection)
|
||||
payload: dict[str, Any] = {
|
||||
"retrieval_trace_id": str(uuid4()),
|
||||
"trigger": "tool",
|
||||
"query": query,
|
||||
"selection": selection,
|
||||
"assistant_selection": assistant_selection,
|
||||
# ``knowledge_base_count`` is retained for older trace consumers. It
|
||||
# historically meant the requested selection count, not an
|
||||
# authorization result, so new consumers must use the explicit fields.
|
||||
"knowledge_base_count": selection_count,
|
||||
"selection_count": selection_count,
|
||||
"authorized_space_count": max(0, int(authorized_space_count)),
|
||||
"mode": mode,
|
||||
"results": result_rows,
|
||||
"total_result_count": len(result_rows),
|
||||
"results_truncated": False,
|
||||
"partial": partial,
|
||||
"warnings": list(dict.fromkeys(warnings or [])),
|
||||
"fallback_knowledge_base_ids": fallback_knowledge_base_ids or [],
|
||||
"knowledge_base_profiles": knowledge_base_profiles or [],
|
||||
"query_plan": query_plan or {},
|
||||
"statistics": statistics,
|
||||
"references": references or [],
|
||||
"diagnostics": diagnostics or {},
|
||||
"timings_ms": {"total": round((perf_counter() - started_at) * 1000, 2)},
|
||||
"zero_result_reason": None if result_rows else zero_result_reason or "KEYWORD_NO_MATCH",
|
||||
}
|
||||
if error:
|
||||
payload["error"] = error
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
|
||||
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
|
||||
|
||||
@@ -39,6 +99,42 @@ def _sanitize_conversation_deposit_value(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _stable_citation_id(item: Mapping[str, Any]) -> str:
|
||||
"""Build a stable cross-call identity without exposing provider IDs."""
|
||||
|
||||
knowledge_base_id = str(
|
||||
item.get("knowledge_base_id") or item.get("base_id") or "unknown"
|
||||
)
|
||||
identity = next(
|
||||
(
|
||||
str(item.get(key) or "").strip()
|
||||
for key in (
|
||||
"chunk_id",
|
||||
"page_ref",
|
||||
"wiki_slug",
|
||||
"slug",
|
||||
"knowledge_id",
|
||||
"id",
|
||||
"title",
|
||||
)
|
||||
if str(item.get(key) or "").strip()
|
||||
),
|
||||
"result",
|
||||
)
|
||||
digest = hashlib.sha256(f"{knowledge_base_id}\0{identity}".encode()).hexdigest()[:20]
|
||||
return f"llmwiki:{digest}"
|
||||
|
||||
|
||||
def _result_score(item: Mapping[str, Any]) -> float:
|
||||
for key in ("relevance_score", "rerank_score", "score", "similarity"):
|
||||
try:
|
||||
value = float(item.get(key))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return value
|
||||
return 0.0
|
||||
|
||||
|
||||
def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
"""Read the conversation selection; ``None`` means legacy/no selection key."""
|
||||
|
||||
@@ -50,6 +146,15 @@ def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str]
|
||||
return list(dict.fromkeys(str(item).strip() for item in raw if str(item).strip()))[:100]
|
||||
|
||||
|
||||
def selected_assistant_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
if not context or "assistant_knowledge_base_ids" not in context:
|
||||
return None
|
||||
raw = context.get("assistant_knowledge_base_ids")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return list(dict.fromkeys(str(item).strip() for item in raw if str(item).strip()))[:100]
|
||||
|
||||
|
||||
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
agent_id = str((context or {}).get("agent_id") or "").strip()
|
||||
if not agent_id or agent_id in {"default", "lead_agent"}:
|
||||
@@ -65,6 +170,7 @@ def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
|
||||
async def llmwiki_search_tool(
|
||||
query: str,
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
knowledge_base_id: str = "",
|
||||
) -> str:
|
||||
"""Search the LLMWiki knowledge selected for this conversation.
|
||||
|
||||
@@ -74,61 +180,247 @@ async def llmwiki_search_tool(
|
||||
|
||||
Args:
|
||||
query: Focused natural-language retrieval query.
|
||||
knowledge_base_id: Optional selected DeerFlow knowledge-base ID. Use one
|
||||
ID per call when several libraries are selected so each library is
|
||||
searched and diagnosed independently.
|
||||
"""
|
||||
|
||||
started_at = perf_counter()
|
||||
|
||||
local_config = get_app_config().llmwiki.local_wiki_index
|
||||
resolved = get_resolved_llmwiki_runtime()
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
|
||||
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
|
||||
|
||||
store = get_llmwiki_store()
|
||||
user_id = get_effective_user_id()
|
||||
is_admin = getattr(get_current_user(), "system_role", None) == "admin"
|
||||
context = runtime.context if runtime.context else {}
|
||||
selected_ids = selected_knowledge_base_ids(context)
|
||||
allowed_ids = _agent_allowed_ids(context)
|
||||
if selected_ids is None:
|
||||
return "No LLMWiki knowledge base is selected for this conversation."
|
||||
elif allowed_ids is not None and not set(selected_ids).issubset(set(allowed_ids)):
|
||||
return "The conversation requested knowledge outside this agent's configured LLMWiki scope."
|
||||
|
||||
if selected_ids == []:
|
||||
return "No LLMWiki knowledge base is selected for this conversation."
|
||||
rows = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(
|
||||
mapping_id,
|
||||
user_id,
|
||||
write=False,
|
||||
is_admin=is_admin,
|
||||
assistant_selected_ids = selected_assistant_knowledge_base_ids(context)
|
||||
selected_ids = selected_ids or []
|
||||
assistant_selected_ids = assistant_selected_ids or []
|
||||
retrieval_mode = str(context.get("knowledge_retrieval_mode") or "classic").strip().lower()
|
||||
if retrieval_mode != "classic":
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=selected_ids,
|
||||
assistant_selection=assistant_selected_ids,
|
||||
zero_result_reason="CLASSIC_MODE_REQUIRED",
|
||||
error="llmwiki_search is available only in classic knowledge retrieval mode.",
|
||||
)
|
||||
allowed_ids = _agent_allowed_ids(context)
|
||||
requested_knowledge_base_id = str(knowledge_base_id or "").strip()
|
||||
if requested_knowledge_base_id:
|
||||
if requested_knowledge_base_id in selected_ids:
|
||||
selected_ids = [requested_knowledge_base_id]
|
||||
assistant_selected_ids = []
|
||||
elif requested_knowledge_base_id in assistant_selected_ids:
|
||||
selected_ids = []
|
||||
assistant_selected_ids = [requested_knowledge_base_id]
|
||||
else:
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=selected_ids,
|
||||
assistant_selection=assistant_selected_ids,
|
||||
zero_result_reason="KNOWLEDGE_SCOPE_DENIED",
|
||||
error="knowledge_base_id is not selected for this conversation.",
|
||||
)
|
||||
if not selected_ids and not assistant_selected_ids:
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=[],
|
||||
assistant_selection=[],
|
||||
zero_result_reason="NO_SELECTED_SPACE",
|
||||
error="No knowledge space is selected for this conversation.",
|
||||
)
|
||||
if row is None:
|
||||
return f"Knowledge base {mapping_id!r} is not available to the current user."
|
||||
rows.append(row)
|
||||
if not rows:
|
||||
return "No authorized LLMWiki knowledge bases are available."
|
||||
|
||||
try:
|
||||
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
|
||||
except (RuntimeError, WikiIndexError) as exc:
|
||||
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
|
||||
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
results = []
|
||||
for item in payload["results"]:
|
||||
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
|
||||
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
|
||||
results.append(visible)
|
||||
return json.dumps(
|
||||
{
|
||||
"query": query,
|
||||
"selection": [row["id"] for row in rows],
|
||||
"knowledge_base_count": len(rows),
|
||||
"mode": payload.get("mode") or "wiki_api",
|
||||
"results": results,
|
||||
"partial": payload.get("partial", False),
|
||||
"warnings": payload.get("warnings", []),
|
||||
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
warnings: list[str] = []
|
||||
partial = False
|
||||
rows: list[dict[str, Any]] = []
|
||||
authorized_assistant_ids: list[str] = []
|
||||
results: list[dict[str, Any]] = []
|
||||
payload: dict[str, Any] = {"mode": "empty", "partial": False, "warnings": []}
|
||||
ordinary_candidate_ids = selected_ids
|
||||
if allowed_ids is not None:
|
||||
allowed_set = set(allowed_ids)
|
||||
denied_ids = [mapping_id for mapping_id in selected_ids if mapping_id not in allowed_set]
|
||||
if denied_ids:
|
||||
partial = True
|
||||
warnings.append("KNOWLEDGE_SCOPE_DENIED")
|
||||
ordinary_candidate_ids = [mapping_id for mapping_id in selected_ids if mapping_id in allowed_set]
|
||||
|
||||
if ordinary_candidate_ids:
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
|
||||
partial = True
|
||||
warnings.append("RETRIEVAL_PROVIDER_UNAVAILABLE")
|
||||
else:
|
||||
try:
|
||||
store = get_llmwiki_store()
|
||||
for mapping_id in ordinary_candidate_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
partial = True
|
||||
warnings.append("SPACE_NOT_FOUND_OR_UNAUTHORIZED")
|
||||
continue
|
||||
rows.append(row)
|
||||
except Exception: # noqa: BLE001 - the assistant channel may still be usable
|
||||
partial = True
|
||||
warnings.append("RETRIEVAL_PROVIDER_UNAVAILABLE")
|
||||
rows = []
|
||||
if rows:
|
||||
try:
|
||||
retrieval_service = get_local_wiki_retrieval_service()
|
||||
try:
|
||||
index_store = get_local_wiki_index_store()
|
||||
except RuntimeError:
|
||||
index_store = None
|
||||
if index_store is None:
|
||||
try:
|
||||
payload = await retrieval_service.search(query.strip(), rows, top_k_pages=20)
|
||||
except TypeError as exc:
|
||||
if "top_k_pages" not in str(exc):
|
||||
raise
|
||||
payload = await retrieval_service.search(query.strip(), rows)
|
||||
payload["knowledge_base_profiles"] = [
|
||||
{
|
||||
"id": row.get("id"),
|
||||
"name": row.get("name"),
|
||||
"description": row.get("description") or "",
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
else:
|
||||
payload = await advanced_search(
|
||||
query=query.strip(),
|
||||
mappings=rows,
|
||||
retrieval_service=retrieval_service,
|
||||
index_store=index_store,
|
||||
top_k=20,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - preserve a successful assistant channel
|
||||
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
|
||||
partial = True
|
||||
warnings.append(code)
|
||||
payload = {"mode": "empty", "results": [], "partial": True, "warnings": [code]}
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
for item in payload.get("results") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
|
||||
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
|
||||
results.append(visible)
|
||||
|
||||
if assistant_selected_ids:
|
||||
from deerflow.persistence.assistant_knowledge import get_assistant_knowledge_store
|
||||
|
||||
assistant_store = get_assistant_knowledge_store()
|
||||
if assistant_store is None:
|
||||
partial = True
|
||||
warnings.append("ASSISTANT_RETRIEVAL_PROVIDER_UNAVAILABLE")
|
||||
assistant_results: list[dict[str, Any]] = []
|
||||
else:
|
||||
get_base = getattr(assistant_store, "get_base", None)
|
||||
if not callable(get_base):
|
||||
partial = True
|
||||
warnings.append("ASSISTANT_RETRIEVAL_PROVIDER_UNAVAILABLE")
|
||||
else:
|
||||
for base_id in assistant_selected_ids:
|
||||
try:
|
||||
base = await get_base(base_id)
|
||||
except Exception: # noqa: BLE001 - keep other spaces/channels searchable
|
||||
partial = True
|
||||
warnings.append("ASSISTANT_RETRIEVAL_FAILED")
|
||||
continue
|
||||
if base is None:
|
||||
partial = True
|
||||
warnings.append("ASSISTANT_SPACE_NOT_FOUND_OR_UNAUTHORIZED")
|
||||
continue
|
||||
authorized_assistant_ids.append(base_id)
|
||||
assistant_results = []
|
||||
if authorized_assistant_ids:
|
||||
try:
|
||||
assistant_results = await assistant_store.search_pages(
|
||||
base_ids=authorized_assistant_ids,
|
||||
query=query.strip(),
|
||||
limit=8,
|
||||
include_chunks=True,
|
||||
embedding_client=get_local_wiki_search_service().embedding,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - preserve a successful ordinary channel
|
||||
partial = True
|
||||
warnings.append("ASSISTANT_RETRIEVAL_FAILED")
|
||||
assistant_results = []
|
||||
for item in assistant_results:
|
||||
results.append(
|
||||
{
|
||||
**item,
|
||||
"knowledge_base_id": item.get("base_id"),
|
||||
"knowledge_base_name": "助手知识库",
|
||||
"type": "assistant_knowledge",
|
||||
"source": "助手知识库",
|
||||
"retrieval_mode": item.get("retrieval_mode") or "hybrid",
|
||||
}
|
||||
)
|
||||
|
||||
payload_warnings = [str(item) for item in payload.get("warnings") or []]
|
||||
warnings.extend(payload_warnings)
|
||||
partial = partial or bool(payload.get("partial", False))
|
||||
raw_diagnostics = payload.get("diagnostics")
|
||||
if isinstance(raw_diagnostics, Mapping):
|
||||
diagnostics = dict(raw_diagnostics)
|
||||
fallback_reason = raw_diagnostics.get("fallback_reason")
|
||||
elif isinstance(raw_diagnostics, list):
|
||||
diagnostics = {"ordinary_searches": raw_diagnostics}
|
||||
fallback_reason = None
|
||||
else:
|
||||
diagnostics = {}
|
||||
fallback_reason = None
|
||||
diagnostics["assistant_candidates"] = sum(1 for item in results if item.get("type") == "assistant_knowledge")
|
||||
diagnostics["ordinary_authorized_space_count"] = len(rows)
|
||||
diagnostics["assistant_authorized_space_count"] = len(authorized_assistant_ids)
|
||||
|
||||
ordinary_results = [item for item in results if item.get("type") != "assistant_knowledge"]
|
||||
assistant_results = [item for item in results if item.get("type") == "assistant_knowledge"]
|
||||
if ordinary_results and assistant_results:
|
||||
mode = "mixed"
|
||||
elif ordinary_results:
|
||||
mode = str(payload.get("mode") or "hybrid")
|
||||
elif assistant_results:
|
||||
assistant_modes = {str(item.get("retrieval_mode") or "hybrid") for item in assistant_results}
|
||||
mode = next(iter(assistant_modes)) if len(assistant_modes) == 1 else "hybrid"
|
||||
else:
|
||||
mode = str(payload.get("mode") or "empty")
|
||||
|
||||
results.sort(key=_result_score, reverse=True)
|
||||
for rank, item in enumerate(results, start=1):
|
||||
item.setdefault("citation_id", _stable_citation_id(item))
|
||||
item.setdefault("relevance_rank", rank)
|
||||
|
||||
if results:
|
||||
zero_result_reason = None
|
||||
elif fallback_reason:
|
||||
zero_result_reason = str(fallback_reason)
|
||||
elif warnings:
|
||||
zero_result_reason = warnings[0] if all(item in {"KNOWLEDGE_SCOPE_DENIED", "SPACE_NOT_FOUND_OR_UNAUTHORIZED", "ASSISTANT_SPACE_NOT_FOUND_OR_UNAUTHORIZED"} for item in warnings) else "RETRIEVAL_PROVIDER_UNAVAILABLE"
|
||||
else:
|
||||
zero_result_reason = "KEYWORD_NO_MATCH"
|
||||
|
||||
return _trace_json(
|
||||
started_at=started_at,
|
||||
query=query,
|
||||
selection=selected_ids,
|
||||
assistant_selection=assistant_selected_ids,
|
||||
authorized_space_count=len(rows) + len(authorized_assistant_ids),
|
||||
mode=mode,
|
||||
results=results,
|
||||
partial=partial,
|
||||
warnings=warnings,
|
||||
fallback_knowledge_base_ids=[str(item) for item in payload.get("fallback_knowledge_base_ids", [])],
|
||||
knowledge_base_profiles=list(payload.get("knowledge_base_profiles") or []),
|
||||
query_plan=dict(payload.get("query_plan") or {}),
|
||||
statistics=payload.get("statistics"),
|
||||
references=list(payload.get("references") or []),
|
||||
diagnostics=diagnostics,
|
||||
zero_result_reason=zero_result_reason,
|
||||
error=("Knowledge retrieval could not complete for any selected space." if warnings and not results else None),
|
||||
)
|
||||
+836
-38
@@ -12,6 +12,7 @@ from langgraph.typing import ContextT
|
||||
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths
|
||||
from deerflow.html_runtime.html_postprocess import apply_mechanical_html_fixes
|
||||
from deerflow.page_reports.templates import (
|
||||
choose_template,
|
||||
get_template_profile,
|
||||
@@ -26,6 +27,141 @@ _MAX_HTML_BYTES = 4 * 1024 * 1024
|
||||
_SAFE_OUTPUT_NAME = re.compile(r'^[^<>:"/\\|?*\x00-\x1f]+\.html$', re.IGNORECASE)
|
||||
_FONT_SIZE_RE = re.compile(r"font-size\s*:\s*([0-9]+(?:\.[0-9]+)?)px", re.IGNORECASE)
|
||||
_MIN_TEXT_FONT_SIZE = 16.0
|
||||
_FLOW_FEATURES = {"flow", "flowchart", "process-flow", "process_flow", "mermaid", "流程图"}
|
||||
_CHART_FEATURES = {"line", "bar", "pie", "radar", "scatter", "gantt", "chart", "echarts", "图表"}
|
||||
_MAP_FEATURES = {"map", "leaflet", "geojson", "geo-json", "地图"}
|
||||
_MULTI_PAGE_INTENT_RE = re.compile(r"多页面|多个页面|多页|分页门户|multi[-\s]?page", re.IGNORECASE)
|
||||
_GEO_MAP_CATALOG = frozenset(
|
||||
{"china-provinces", "us-states", "japan-prefectures", "taiwan-counties"}
|
||||
)
|
||||
_SCRIPT_STYLE_COMMENT_RE = re.compile(
|
||||
r"<style\b[^>]*>[\s\S]*?</style>|<!--[\s\S]*?-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SCRIPT_CONTENT_RE = re.compile(r"(<script\b[^>]*>)[\s\S]*?</script>", re.IGNORECASE)
|
||||
_OPEN_PORTAL_ELEMENT_RE = re.compile(r"<(?:main|section|article|div)\b[^>]*>", re.IGNORECASE)
|
||||
_OPEN_ANCHOR_RE = re.compile(r"<a\b[^>]*>", re.IGNORECASE)
|
||||
_DARK_THEME_RE = re.compile(
|
||||
r"(?:\bdata(?:-df)?-theme\s*=\s*[\"'][^\"']*\bdark\b|"
|
||||
r"--(?:bg|background|surface)(?:-[\w-]+)?\s*:\s*#[0-4][0-9a-f]{5}\b)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _required_visualizations(features: Iterable[str], material_brief: str = "") -> set[str]:
|
||||
"""Normalize explicit features and textual source hints into runtime contracts."""
|
||||
normalized = {str(value).strip().lower() for value in features if str(value).strip()}
|
||||
brief = (material_brief or "").lower()
|
||||
required: set[str] = set()
|
||||
if normalized & _FLOW_FEATURES or re.search(r"流程图|flowchart|process[-_\s]?flow|```mermaid", brief):
|
||||
required.add("flowchart")
|
||||
if normalized & _CHART_FEATURES or re.search(r"图表|折线图|柱状图|饼图|雷达图|散点图|甘特图|echarts", brief):
|
||||
required.add("chart")
|
||||
if normalized & _MAP_FEATURES or re.search(r"地图|leaflet|经纬度|坐标点", brief):
|
||||
required.add("map")
|
||||
return required
|
||||
|
||||
|
||||
def _active_html_map_engine() -> str:
|
||||
from deerflow.html_runtime.map_engine import html_map_engine
|
||||
|
||||
return html_map_engine()
|
||||
|
||||
|
||||
def _visualization_contract(required: set[str]) -> dict[str, object]:
|
||||
leaflet = _active_html_map_engine() == "leaflet"
|
||||
map_contract: dict[str, object] = (
|
||||
{
|
||||
"renderer": "Leaflet 地图网页",
|
||||
"required_assets": ["leaflet.css", "leaflet.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-map\"", "data-df-map"],
|
||||
"forbidden": ["data-df-geo", "df-geo-map", "manual L.map", "tile URL in HTML"],
|
||||
}
|
||||
if leaflet
|
||||
else {
|
||||
"renderer": "ECharts GeoJSON(Canvas)",
|
||||
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-geo-map\"", "data-df-geo"],
|
||||
"supported_maps": sorted(_GEO_MAP_CATALOG),
|
||||
"forbidden": ["Leaflet", "data-df-map", "map tile URL", "manual echarts.init"],
|
||||
}
|
||||
)
|
||||
contracts: dict[str, dict[str, object]] = {
|
||||
"flowchart": {
|
||||
"renderer": "AntV G6",
|
||||
"required_assets": ["g6.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-flow\"", "data-df-flow"],
|
||||
"forbidden": ["Mermaid in HTML", "handwritten HTML/CSS flow nodes", "new G6.Graph"],
|
||||
},
|
||||
"chart": {
|
||||
"renderer": "ECharts",
|
||||
"required_assets": ["echarts.min.js", "deerflow-runtime.js"],
|
||||
"required_markup": ["class=\"df-chart\"", "data-df-chart"],
|
||||
},
|
||||
"map": map_contract,
|
||||
}
|
||||
return {name: contracts[name] for name in sorted(required)}
|
||||
|
||||
|
||||
def _requested_page_count(preferred_page_count: int | None, page_brief: str = "") -> int:
|
||||
"""Default to one page, or a three-page portal for explicit multi-page intent."""
|
||||
requested = max(1, int(preferred_page_count)) if preferred_page_count is not None else 1
|
||||
if requested > 1:
|
||||
return requested
|
||||
# Agents occasionally omit preferred_page_count even though their persisted
|
||||
# brief says the user explicitly requested a multi-page portal. Keep that
|
||||
# user intent authoritative and give the portal a useful, predictable
|
||||
# default split: overview, primary analysis, and detail.
|
||||
if _MULTI_PAGE_INTENT_RE.search(page_brief or ""):
|
||||
return 3
|
||||
return requested
|
||||
|
||||
|
||||
def _requested_layout_mode(page_brief: str) -> str:
|
||||
"""Use a fixed-screen template only when the user explicitly asks for one."""
|
||||
brief = (page_brief or "").lower()
|
||||
if re.search(r"大屏|驾驶舱|单屏|固定画布|1920\s*[x×*]\s*1080|big[-\s]?screen", brief):
|
||||
return "screen"
|
||||
return "document"
|
||||
|
||||
|
||||
def _choose_page_profile(
|
||||
*,
|
||||
theme: str,
|
||||
content_density: str,
|
||||
required_features: list[str],
|
||||
requested_page_count: int,
|
||||
selection_key: str,
|
||||
exclude_template_ids: list[str],
|
||||
template_id: str | None,
|
||||
layout_mode: str = "document",
|
||||
):
|
||||
multi_page = requested_page_count > 1
|
||||
if not multi_page and layout_mode == "document":
|
||||
normalized_theme = (theme or "").strip().lower()
|
||||
document_id = "dark-document" if normalized_theme in {"dark", "deep-blue", "dark-blue"} else "light-document"
|
||||
return get_template_profile(document_id)
|
||||
explicit = get_template_profile(template_id) if (template_id or "").strip() else None
|
||||
if explicit is not None and (explicit.page_count > 1) == multi_page:
|
||||
return explicit
|
||||
|
||||
wrong_mode_ids = {
|
||||
profile.id
|
||||
for profile in list_template_profiles()
|
||||
if (profile.page_count > 1) != multi_page
|
||||
}
|
||||
excluded = sorted(
|
||||
wrong_mode_ids
|
||||
| {value.strip().lower() for value in exclude_template_ids if value.strip()}
|
||||
)
|
||||
return choose_template(
|
||||
theme=theme,
|
||||
content_density=content_density,
|
||||
required_features=required_features,
|
||||
preferred_page_count=requested_page_count,
|
||||
selection_key=selection_key,
|
||||
exclude_template_ids=excluded,
|
||||
)
|
||||
|
||||
|
||||
def _thread_id(runtime: ToolRuntime[ContextT, ThreadState]) -> str:
|
||||
@@ -68,12 +204,11 @@ def _apply_minimum_font_size(html: str) -> str:
|
||||
"""Normalize editable template copies to the report designer's font floor.
|
||||
|
||||
The library includes some legacy dashboard skeletons with compact labels.
|
||||
They are useful layout starting points, but the designer contract requires
|
||||
that visible text is at least 16px. Normalize the *per-thread copy* here
|
||||
so a weaker model does not have to remember a large CSS-only cleanup before
|
||||
it can work on the actual report content. The immutable source template is
|
||||
never changed, and ``validate`` remains the final guard if later edits add
|
||||
a smaller declaration.
|
||||
They are useful layout starting points, but visible text should start at
|
||||
least 16px. Normalize the *per-thread copy* here so the model writes
|
||||
against a compliant baseline instead of discovering small type later.
|
||||
The immutable source template is never changed. Validate no longer
|
||||
rejects leftover small ``font-size`` declarations.
|
||||
"""
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
@@ -106,6 +241,21 @@ def _edit_contract(template_id: str) -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
def _apply_document_theme(html: str, theme: str) -> str:
|
||||
"""Set the shared responsive document template's requested color theme."""
|
||||
if theme not in {"light", "dark"}:
|
||||
return html
|
||||
if re.search(r"<html\b[^>]*\bdata-theme=", html, flags=re.IGNORECASE):
|
||||
return re.sub(
|
||||
r"(<html\b[^>]*\bdata-theme\s*=\s*)[\"'][^\"']*[\"']",
|
||||
rf'\1"{theme}"',
|
||||
html,
|
||||
count=1,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
return re.sub(r"<html\b", f'<html data-theme="{theme}"', html, count=1, flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def _prepare(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
*,
|
||||
@@ -119,37 +269,54 @@ def _prepare(
|
||||
output_name: str,
|
||||
page_brief: str,
|
||||
) -> str:
|
||||
profile = (
|
||||
get_template_profile(template_id)
|
||||
if (template_id or "").strip()
|
||||
else choose_template(
|
||||
theme=theme,
|
||||
content_density=content_density,
|
||||
required_features=required_features,
|
||||
preferred_page_count=preferred_page_count,
|
||||
selection_key=selection_key,
|
||||
exclude_template_ids=exclude_template_ids,
|
||||
)
|
||||
required_visualizations = _required_visualizations(required_features, page_brief)
|
||||
requested_page_count = _requested_page_count(preferred_page_count, page_brief)
|
||||
layout_mode = _requested_layout_mode(page_brief)
|
||||
profile = _choose_page_profile(
|
||||
theme=theme,
|
||||
content_density=content_density,
|
||||
required_features=required_features,
|
||||
requested_page_count=requested_page_count,
|
||||
selection_key=selection_key,
|
||||
exclude_template_ids=exclude_template_ids,
|
||||
template_id=template_id,
|
||||
layout_mode=layout_mode,
|
||||
)
|
||||
source = resolve_template_file(profile.id)
|
||||
thread_id, user_id, workspace, outputs = _thread_directories(runtime)
|
||||
filename = _safe_output_filename(output_name)
|
||||
working_copy = workspace / f"{Path(filename).stem}--{profile.id}.html"
|
||||
# Keep the editable copy in outputs from the beginning. The frontend live
|
||||
# preview intentionally watches output writes, while workspace files are
|
||||
# treated as hidden helpers. This makes the sandbox open while HTML is
|
||||
# being generated instead of only after the final present_files call.
|
||||
working_copy = outputs / filename
|
||||
brief_path = workspace / f"{Path(filename).stem}--page-brief.json"
|
||||
# Copy first, then normalize only the editable thread-local copy. This
|
||||
# preserves the bundled assets exactly as supplied while giving every run a
|
||||
# usable 16px-or-larger starting point.
|
||||
working_copy.write_text(_apply_minimum_font_size(source.read_text(encoding="utf-8")), encoding="utf-8")
|
||||
template_html = _apply_minimum_font_size(source.read_text(encoding="utf-8"))
|
||||
template_html = _apply_document_theme(template_html, profile.theme)
|
||||
working_copy.write_text(template_html, encoding="utf-8")
|
||||
brief = {
|
||||
"template": profile.to_dict(),
|
||||
"requested_theme": theme or "auto",
|
||||
"requested_content_density": content_density or "balanced",
|
||||
"requested_page_count": requested_page_count,
|
||||
"page_mode": "multi" if requested_page_count > 1 else "single",
|
||||
"layout_mode": layout_mode,
|
||||
"required_features": required_features,
|
||||
"recommended_final_output": _virtual_path(outputs / filename, thread_id=thread_id, user_id=user_id),
|
||||
"recommended_final_output": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
|
||||
"working_copy": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
|
||||
"user_material_brief": (page_brief or "").strip(),
|
||||
"required_visualizations": sorted(required_visualizations),
|
||||
"visualization_contract": _visualization_contract(required_visualizations),
|
||||
"edit_contract": _edit_contract(profile.id),
|
||||
"font_rule": "所有可见文字最小字号为 16px;生成后必须通过 validate 校验。",
|
||||
"font_rule": "所有可见文字最小字号为 16px;写 HTML 时直接达到,不再单独做字号校验。",
|
||||
"layout_rule": (
|
||||
"普通单页报告必须是可纵向滚动的响应式文档;禁止固定 1920×1080 舞台、body overflow:hidden 和视口缩放脚本。"
|
||||
if layout_mode == "document"
|
||||
else "用户明确要求大屏,可使用固定单屏布局,但仍需保证各区块不重叠。"
|
||||
),
|
||||
}
|
||||
brief_path.write_text(json.dumps(brief, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return json.dumps(
|
||||
@@ -159,7 +326,11 @@ def _prepare(
|
||||
"working_copy": _virtual_path(working_copy, thread_id=thread_id, user_id=user_id),
|
||||
"brief_path": _virtual_path(brief_path, thread_id=thread_id, user_id=user_id),
|
||||
"final_output": brief["recommended_final_output"],
|
||||
"next_step": "读取 brief_path 和 working_copy,改完后写入 final_output,再调用 action=validate。",
|
||||
"next_step": (
|
||||
"读取 brief_path 和 working_copy,严格实现 visualization_contract,"
|
||||
"直接编辑 working_copy(它同时也是 final_output,可实时打开沙箱预览),"
|
||||
"完成后调用 action=validate;不要再用 cp/bash 复制文件。"
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -187,10 +358,548 @@ def _count_textual_content(html: str) -> int:
|
||||
return len(re.sub(r"\s+", "", plain))
|
||||
|
||||
|
||||
def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -> dict[str, object]:
|
||||
def _brief_visualizations(actual: Path, workspace: Path) -> set[str]:
|
||||
"""Recover prepare-time visualization requirements for final validation."""
|
||||
stems = [actual.stem]
|
||||
if "--" in actual.stem:
|
||||
stems.append(actual.stem.split("--", 1)[0])
|
||||
for stem in dict.fromkeys(stems):
|
||||
brief_path = workspace / f"{stem}--page-brief.json"
|
||||
if not brief_path.is_file():
|
||||
continue
|
||||
try:
|
||||
brief = json.loads(brief_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
declared = brief.get("required_visualizations", [])
|
||||
features = brief.get("required_features", [])
|
||||
material = brief.get("user_material_brief", "")
|
||||
values = [*(declared if isinstance(declared, list) else []), *(features if isinstance(features, list) else [])]
|
||||
return _required_visualizations(values, material if isinstance(material, str) else "")
|
||||
return set()
|
||||
|
||||
|
||||
def _brief_page_count(actual: Path, workspace: Path) -> int | None:
|
||||
stems = [actual.stem]
|
||||
if "--" in actual.stem:
|
||||
stems.append(actual.stem.split("--", 1)[0])
|
||||
for stem in dict.fromkeys(stems):
|
||||
brief_path = workspace / f"{stem}--page-brief.json"
|
||||
if not brief_path.is_file():
|
||||
continue
|
||||
try:
|
||||
brief = json.loads(brief_path.read_text(encoding="utf-8"))
|
||||
value = int(brief.get("requested_page_count", 1))
|
||||
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
||||
continue
|
||||
return max(1, value)
|
||||
return None
|
||||
|
||||
|
||||
def _brief_layout_mode(actual: Path, workspace: Path) -> str | None:
|
||||
stems = [actual.stem]
|
||||
if "--" in actual.stem:
|
||||
stems.append(actual.stem.split("--", 1)[0])
|
||||
for stem in dict.fromkeys(stems):
|
||||
brief_path = workspace / f"{stem}--page-brief.json"
|
||||
if not brief_path.is_file():
|
||||
continue
|
||||
try:
|
||||
brief = json.loads(brief_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
value = str(brief.get("layout_mode", "")).strip().lower()
|
||||
if value in {"document", "screen"}:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _count_page_views(html: str) -> int:
|
||||
patterns = (
|
||||
r"\bdata-page\s*=",
|
||||
r"\brole\s*=\s*[\"']tabpanel[\"']",
|
||||
r"class\s*=\s*[\"'][^\"']*\breport-page\b[^\"']*[\"']",
|
||||
r"class\s*=\s*[\"'][^\"']*\bview\b[^\"']*[\"']",
|
||||
)
|
||||
return max(1, *(len(re.findall(pattern, html, flags=re.IGNORECASE)) for pattern in patterns))
|
||||
|
||||
|
||||
def _visible_markup(html: str) -> str:
|
||||
"""Remove non-rendered blocks before inspecting portal navigation markup."""
|
||||
markup = _SCRIPT_STYLE_COMMENT_RE.sub("", html)
|
||||
return _SCRIPT_CONTENT_RE.sub(r"\1</script>", markup)
|
||||
|
||||
|
||||
def _tag_attribute(tag: str, name: str) -> str | None:
|
||||
match = re.search(
|
||||
rf"(?<![-\w]){re.escape(name)}\s*=\s*([\"'])(.*?)\1",
|
||||
tag,
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
return match.group(2).strip() if match else None
|
||||
|
||||
|
||||
def _tag_has_class(tag: str, class_name: str) -> bool:
|
||||
classes = _tag_attribute(tag, "class")
|
||||
return bool(classes and class_name in classes.split())
|
||||
|
||||
|
||||
def _linked_asset_exists(markup: str, *, tag_name: str, attribute: str, filename: str) -> bool:
|
||||
return bool(
|
||||
re.search(
|
||||
rf"<{tag_name}\b[^>]*\b{attribute}\s*=\s*[\"'][^\"']*"
|
||||
rf"{re.escape(filename)}(?:[?#][^\"']*)?[\"'][^>]*>",
|
||||
markup,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _logical_portal_page_ids(markup: str) -> tuple[list[str], int]:
|
||||
page_ids: list[str] = []
|
||||
missing_id_count = 0
|
||||
for tag in _OPEN_PORTAL_ELEMENT_RE.findall(markup):
|
||||
if not (_tag_has_class(tag, "page") and _tag_attribute(tag, "data-page") is not None):
|
||||
continue
|
||||
page_id = _tag_attribute(tag, "id")
|
||||
if page_id:
|
||||
page_ids.append(page_id)
|
||||
else:
|
||||
missing_id_count += 1
|
||||
return page_ids, missing_id_count
|
||||
|
||||
|
||||
def _multi_page_portal_errors(html: str, page_views: int) -> list[str]:
|
||||
"""Require the browser-first portal contract for explicit multi-page HTML."""
|
||||
requirements = (
|
||||
("link", "href", "bootstrap.min.css", "Bootstrap 样式 bootstrap.min.css"),
|
||||
("script", "src", "alpine.min.js", "Alpine 运行库 alpine.min.js"),
|
||||
("script", "src", "bootstrap.bundle.min.js", "Bootstrap 交互运行库 bootstrap.bundle.min.js"),
|
||||
)
|
||||
markup = _visible_markup(html)
|
||||
missing = [
|
||||
label
|
||||
for tag_name, attribute, filename, label in requirements
|
||||
if not _linked_asset_exists(markup, tag_name=tag_name, attribute=attribute, filename=filename)
|
||||
]
|
||||
errors: list[str] = []
|
||||
if missing:
|
||||
errors.append("多页面轻量门户缺少:" + "、".join(missing) + "。")
|
||||
if not any(_tag_has_class(tag, "pages") for tag in _OPEN_PORTAL_ELEMENT_RE.findall(markup)):
|
||||
errors.append("多页面轻量门户必须使用 `.pages` 容器组织逻辑页面。")
|
||||
page_ids, missing_id_count = _logical_portal_page_ids(markup)
|
||||
if page_views < 2 or len(page_ids) + missing_id_count < 2:
|
||||
errors.append("多页面轻量门户必须至少包含两个带 `data-page` 的逻辑页面。")
|
||||
if missing_id_count:
|
||||
errors.append("每个带 `data-page` 的逻辑页面都必须有唯一的 `id=\"page-...\"`。")
|
||||
elif len(set(page_ids)) != len(page_ids):
|
||||
errors.append("多页面轻量门户的逻辑页面 id 必须唯一。")
|
||||
|
||||
nav_targets: list[str] = []
|
||||
for tag in _OPEN_ANCHOR_RE.findall(markup):
|
||||
href = _tag_attribute(tag, "href")
|
||||
if href and href.startswith("#page-"):
|
||||
nav_targets.append(href[1:])
|
||||
if not nav_targets:
|
||||
errors.append("多页面轻量门户导航必须使用 `href=\"#page-...\"` 锚点。")
|
||||
else:
|
||||
page_id_set = set(page_ids)
|
||||
invalid_targets = sorted(set(nav_targets) - page_id_set)
|
||||
if invalid_targets:
|
||||
errors.append(
|
||||
"多页面导航锚点没有对应逻辑页面:" + "、".join(f"#{target}" for target in invalid_targets) + "。"
|
||||
)
|
||||
unlinked_pages = sorted(page_id_set - set(nav_targets))
|
||||
if unlinked_pages:
|
||||
errors.append("逻辑页面缺少导航入口:" + "、".join(f"#{page_id}" for page_id in unlinked_pages) + "。")
|
||||
|
||||
page_id_set = set(page_ids)
|
||||
hash_targets = re.findall(
|
||||
r"(?:window\.)?location\.hash\s*=\s*[\"']#([^\"']+)[\"']",
|
||||
html,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
invalid_hash_targets = sorted(set(hash_targets) - page_id_set)
|
||||
if invalid_hash_targets:
|
||||
errors.append(
|
||||
"脚本切页 hash 没有对应逻辑页面:"
|
||||
+ "、".join(f"#{target}" for target in invalid_hash_targets)
|
||||
+ "。"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _append_missing_contract_errors(errors: list[str], lowered: str, kind: str) -> None:
|
||||
requirements = {
|
||||
"flowchart": (
|
||||
("g6.min.js", "G6 浏览器运行库 g6.min.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-flow", "data-df-flow JSON 数据块"),
|
||||
("df-flow", "df-flow 流程图容器"),
|
||||
),
|
||||
"chart": (
|
||||
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-chart", "data-df-chart JSON 数据块"),
|
||||
("df-chart", "df-chart 图表容器"),
|
||||
),
|
||||
"map": (
|
||||
(
|
||||
("leaflet.css", "Leaflet 样式 leaflet.css"),
|
||||
("leaflet.js", "Leaflet 浏览器运行库 leaflet.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-map", "data-df-map JSON 数据块"),
|
||||
('class="df-map"', "df-map 地图容器"),
|
||||
)
|
||||
if _active_html_map_engine() == "leaflet"
|
||||
else (
|
||||
("echarts.min.js", "ECharts 浏览器运行库 echarts.min.js"),
|
||||
("deerflow-runtime.js", "统一初始化脚本 deerflow-runtime.js"),
|
||||
("data-df-geo", "data-df-geo JSON 数据块"),
|
||||
("df-geo-map", "df-geo-map 地图容器"),
|
||||
)
|
||||
),
|
||||
}
|
||||
labels = {"flowchart": "流程图", "chart": "图表", "map": "地图"}
|
||||
missing = [description for marker, description in requirements[kind] if marker not in lowered]
|
||||
if missing:
|
||||
errors.append(f"{labels[kind]}未使用平台绘图契约,缺少:{'、'.join(missing)}。")
|
||||
|
||||
|
||||
def _flow_layout_errors(html: str) -> list[str]:
|
||||
"""Catch undersized long G6 diagrams before they overlap or become tiny."""
|
||||
errors: list[str] = []
|
||||
pattern = re.compile(
|
||||
r"<script\b[^>]*data-df-flow\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
for flow_id, payload in pattern.findall(html):
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
errors.append(f"流程图 {flow_id} 的 data-df-flow 不是严格 JSON。")
|
||||
continue
|
||||
nodes = data.get("nodes", []) if isinstance(data, dict) else []
|
||||
node_count = len(nodes) if isinstance(nodes, list) else 0
|
||||
tag = re.search(
|
||||
rf"<[^>]+\bid\s*=\s*[\"']{re.escape(flow_id)}[\"'][^>]*>",
|
||||
html,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
height = None
|
||||
if tag:
|
||||
height_match = re.search(r"height\s*:\s*([0-9]+(?:\.[0-9]+)?)px", tag.group(0), flags=re.IGNORECASE)
|
||||
if height_match:
|
||||
height = float(height_match.group(1))
|
||||
if height is None:
|
||||
css_height = re.search(
|
||||
rf"#{re.escape(flow_id)}\s*\{{[^}}]*height\s*:\s*([0-9]+(?:\.[0-9]+)?)px",
|
||||
html,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
if css_height:
|
||||
height = float(css_height.group(1))
|
||||
if height is None:
|
||||
errors.append(f"流程图 {flow_id} 必须设置明确的像素高度。")
|
||||
elif height < 360:
|
||||
errors.append(f"流程图 {flow_id} 高度仅 {height:g}px;至少需要 360px。")
|
||||
if node_count > 8:
|
||||
direction = str(data.get("direction", "LR")).upper() if isinstance(data, dict) else "LR"
|
||||
if direction != "TB":
|
||||
errors.append(f"流程图 {flow_id} 有 {node_count} 个节点;请拆分,或改用 TB 纵向布局。")
|
||||
if height is not None and height < 600:
|
||||
errors.append(f"流程图 {flow_id} 有 {node_count} 个节点;TB 布局高度至少需要 600px。")
|
||||
return errors
|
||||
|
||||
|
||||
def _geo_map_errors(html: str) -> list[str]:
|
||||
"""Validate the packaged GeoJSON map contract without loading a browser."""
|
||||
errors: list[str] = []
|
||||
pattern = re.compile(
|
||||
r"<script\b[^>]*data-df-geo\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
for map_id, payload in pattern.findall(html):
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
errors.append(f"地图 {map_id} 的 data-df-geo 不是严格 JSON。")
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
errors.append(f"地图 {map_id} 的 data-df-geo 必须是 JSON 对象。")
|
||||
continue
|
||||
if data.get("map") not in _GEO_MAP_CATALOG:
|
||||
errors.append(
|
||||
f"地图 {map_id} 必须使用内置地图之一:{', '.join(sorted(_GEO_MAP_CATALOG))};"
|
||||
"不要引用外部地图、Leaflet 或地图瓦片。"
|
||||
)
|
||||
invalid_descriptions: list[str] = []
|
||||
for key in ("regions", "markers", "lines"):
|
||||
if key in data and not isinstance(data[key], list):
|
||||
errors.append(f"地图 {map_id} 的 {key} 必须是数组。")
|
||||
continue
|
||||
for index, item in enumerate(data.get(key, [])):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
description = item.get("description")
|
||||
if isinstance(description, str) and re.search(
|
||||
r"<\s*/?\s*[a-zA-Z][^>]*>|\bon\w+\s*=",
|
||||
description,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
invalid_descriptions.append(f"{key}[{index}]")
|
||||
if invalid_descriptions:
|
||||
errors.append(
|
||||
f"地图 {map_id} 的 {', '.join(invalid_descriptions)}.description 必须是纯文本;"
|
||||
"地图运行时会转义 HTML。请监听 deerflow:geo-map-click 实现页面联动。"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _contains_chart_callback(value: object) -> bool:
|
||||
"""Reject callback-shaped values: declarative chart payloads must stay JSON-only."""
|
||||
if isinstance(value, str):
|
||||
return bool(re.search(r"(?:\bfunction\s*\(|=>)", value))
|
||||
if isinstance(value, dict):
|
||||
return any(_contains_chart_callback(item) for item in value.values())
|
||||
if isinstance(value, list):
|
||||
return any(_contains_chart_callback(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _is_dark_chart_page(html: str) -> bool:
|
||||
return bool(_DARK_THEME_RE.search(re.sub(r"/\*[\s\S]*?\*/", "", html)))
|
||||
|
||||
|
||||
def _is_dark_text_color(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return True
|
||||
color = value.strip()
|
||||
match = re.fullmatch(r"#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})", color)
|
||||
if not match:
|
||||
return False
|
||||
hex_color = match.group(1)
|
||||
if len(hex_color) == 3:
|
||||
hex_color = "".join(character * 2 for character in hex_color)
|
||||
red, green, blue = (int(hex_color[index : index + 2], 16) for index in range(0, 6, 2))
|
||||
luminance = (0.2126 * red + 0.7152 * green + 0.0722 * blue) / 255
|
||||
return luminance < 0.35
|
||||
|
||||
|
||||
def _has_readable_color(style: object) -> bool:
|
||||
return isinstance(style, dict) and "color" in style and not _is_dark_text_color(style["color"])
|
||||
|
||||
|
||||
def _as_option_list(value: object) -> list[dict[str, object]]:
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _has_markup_class(html: str, class_name: str) -> bool:
|
||||
return bool(
|
||||
re.search(
|
||||
rf"\bclass\s*=\s*[\"'][^\"']*\b{re.escape(class_name)}\b",
|
||||
_visible_markup(html),
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _css_selector_has_color(html: str, selector: str) -> bool:
|
||||
for selectors, declarations in re.findall(r"([^{}]+)\{([^{}]*)\}", html, flags=re.DOTALL):
|
||||
if re.search(selector, selectors, flags=re.IGNORECASE) and re.search(
|
||||
r"\bcolor\s*:", declarations, flags=re.IGNORECASE
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _dark_component_contrast_errors(html: str) -> list[str]:
|
||||
"""Catch Bootstrap light-token leakage into dark cards, tables, and modals."""
|
||||
if not _is_dark_chart_page(html):
|
||||
return []
|
||||
errors: list[str] = []
|
||||
requirements = (
|
||||
("card", r"\.card(?![-\w])", "深色页面的 Bootstrap `.card` 必须显式设置浅色 `color`。"),
|
||||
(
|
||||
"data-table",
|
||||
r"\.data-table(?![-\w])(?=\s*(?:,|$)|\s+td\b)",
|
||||
"深色页面的明细表必须为 `.data-table` 或 `.data-table td` 设置浅色 `color`。",
|
||||
),
|
||||
(
|
||||
"modal",
|
||||
r"\.modal-(?:content|body)(?![-\w])",
|
||||
"深色页面的 Bootstrap 弹窗必须为 `.modal-content` 或 `.modal-body` 设置浅色 `color`。",
|
||||
),
|
||||
)
|
||||
for markup_class, selector, error in requirements:
|
||||
if _has_markup_class(html, markup_class) and not _css_selector_has_color(html, selector):
|
||||
errors.append(error)
|
||||
return errors
|
||||
|
||||
|
||||
def _dark_chart_contrast_errors(chart_id: str, data: dict[str, object]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _has_readable_color(data.get("textStyle")):
|
||||
errors.append(f"深色页面的图表 {chart_id} 必须设置可读的 textStyle.color。")
|
||||
|
||||
title = data.get("title")
|
||||
if isinstance(title, dict) and title.get("text") and not _has_readable_color(title.get("textStyle")):
|
||||
errors.append(f"深色页面的图表 {chart_id} 标题必须设置 title.textStyle.color。")
|
||||
|
||||
legend = data.get("legend")
|
||||
if isinstance(legend, dict) and legend.get("show") is not False and not _has_readable_color(legend.get("textStyle")):
|
||||
errors.append(f"深色页面的图表 {chart_id} 图例必须设置 legend.textStyle.color。")
|
||||
|
||||
for axis_name in ("xAxis", "yAxis"):
|
||||
for index, axis in enumerate(_as_option_list(data.get(axis_name))):
|
||||
axis_label = axis.get("axisLabel")
|
||||
if not (isinstance(axis_label, dict) and axis_label.get("show") is False) and not _has_readable_color(axis_label):
|
||||
errors.append(
|
||||
f"深色页面的图表 {chart_id} 的 {axis_name}[{index}].axisLabel 必须设置可读 color。"
|
||||
)
|
||||
if axis.get("name") and not _has_readable_color(axis.get("nameTextStyle")):
|
||||
errors.append(
|
||||
f"深色页面的图表 {chart_id} 的 {axis_name}[{index}].nameTextStyle 必须设置可读 color。"
|
||||
)
|
||||
|
||||
for index, series in enumerate(_as_option_list(data.get("series"))):
|
||||
series_type = str(series.get("type") or "").lower()
|
||||
label = series.get("label")
|
||||
labels_are_visible = series_type in {"pie", "funnel", "radar"} or not (
|
||||
isinstance(label, dict) and label.get("show") is False
|
||||
)
|
||||
if labels_are_visible and isinstance(label, dict) and not _has_readable_color(label):
|
||||
errors.append(f"深色页面的图表 {chart_id} 的 series[{index}].label 必须设置可读 color。")
|
||||
elif labels_are_visible and series_type in {"pie", "funnel", "radar"} and not isinstance(label, dict):
|
||||
errors.append(f"深色页面的图表 {chart_id} 的 {series_type} 系列必须设置 label.color。")
|
||||
return errors
|
||||
|
||||
|
||||
def _chart_errors(html: str) -> list[str]:
|
||||
"""Validate declarative ECharts payloads before the browser receives them."""
|
||||
errors: list[str] = []
|
||||
pattern = re.compile(
|
||||
r"<script\b[^>]*data-df-chart\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
for chart_id, payload in pattern.findall(html):
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
hint = (
|
||||
f"图表 {chart_id} 的 data-df-chart 不是严格 JSON;"
|
||||
"不要写 function/回调或其他 JavaScript 表达式。"
|
||||
)
|
||||
if _is_dark_chart_page(html):
|
||||
hint += (
|
||||
" 修复该 JSON 时把深色可读色一次性写齐:"
|
||||
"textStyle、title.textStyle、legend.textStyle、"
|
||||
"xAxis/yAxis.axisLabel、nameTextStyle、series.label 使用 #e8f4ff 或 #b8c8d8。"
|
||||
)
|
||||
errors.append(hint)
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
errors.append(f"图表 {chart_id} 的 data-df-chart 必须是 JSON 对象。")
|
||||
continue
|
||||
if _contains_chart_callback(data):
|
||||
errors.append(
|
||||
f"图表 {chart_id} 的 data-df-chart 包含函数或箭头函数;"
|
||||
"请改用字面量颜色、数据项 itemStyle 或字符串 formatter。"
|
||||
)
|
||||
if _is_dark_chart_page(html):
|
||||
errors.extend(_dark_chart_contrast_errors(chart_id, data))
|
||||
return errors
|
||||
|
||||
|
||||
def _plain_description_errors(map_id: str, data: dict) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for key in ("regions", "markers", "lines", "polygons"):
|
||||
if key in data and not isinstance(data[key], list):
|
||||
errors.append(f"地图 {map_id} 的 {key} 必须是数组。")
|
||||
continue
|
||||
for item in data.get(key, []):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
description = item.get("description")
|
||||
if isinstance(description, str) and re.search(
|
||||
r"<\s*/?\s*[a-zA-Z][^>]*>|\bon\w+\s*=",
|
||||
description,
|
||||
):
|
||||
errors.append(f"地图 {map_id} 的 description 必须是纯文本,不能包含 HTML 或事件属性。")
|
||||
break
|
||||
return errors
|
||||
|
||||
|
||||
def _leaflet_map_errors(html: str) -> list[str]:
|
||||
"""Validate the Leaflet map-webpage contract without loading a browser."""
|
||||
errors: list[str] = []
|
||||
pattern = re.compile(
|
||||
r"<script\b[^>]*data-df-map\s*=\s*[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</script>",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
for map_id, payload in pattern.findall(html):
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
errors.append(f"地图 {map_id} 的 data-df-map 不是严格 JSON。")
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
errors.append(f"地图 {map_id} 的 data-df-map 必须是 JSON 对象。")
|
||||
continue
|
||||
for marker in data.get("markers") or []:
|
||||
if not isinstance(marker, dict):
|
||||
continue
|
||||
coordinate = marker.get("coordinate")
|
||||
if not isinstance(coordinate, list) or len(coordinate) < 2:
|
||||
errors.append(f"地图 {map_id} 的标点 {marker.get('name') or ''} 缺少 [经度,纬度]。")
|
||||
continue
|
||||
try:
|
||||
lng, lat = float(coordinate[0]), float(coordinate[1])
|
||||
except (TypeError, ValueError):
|
||||
errors.append(f"地图 {map_id} 的标点坐标必须是数字。")
|
||||
continue
|
||||
if not (-180 <= lng <= 180 and -90 <= lat <= 90):
|
||||
errors.append(f"地图 {map_id} 的标点坐标超出经纬度范围。")
|
||||
errors.extend(_plain_description_errors(map_id, data))
|
||||
return errors
|
||||
|
||||
|
||||
def _map_contract_errors(html: str) -> list[str]:
|
||||
"""Reject the map engine that the current setting does not allow."""
|
||||
lowered = html.lower()
|
||||
errors: list[str] = []
|
||||
if _active_html_map_engine() == "leaflet":
|
||||
if any(marker in lowered for marker in ("data-df-geo", "df-geo-map", "echarts.registermap")):
|
||||
errors.append("当前 HTML 地图引擎是地图网页,请使用 Leaflet 的 data-df-map,不要使用 data-df-geo。")
|
||||
if re.search(r"\bl\s*\.\s*map\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 Leaflet;请只提供 data-df-map JSON 数据块。")
|
||||
errors.extend(_leaflet_map_errors(html))
|
||||
return errors
|
||||
if any(marker in lowered for marker in ("leaflet", "data-df-map", "tilelayer", "map_tile_url")):
|
||||
errors.append("HTML 网页地图禁止使用 Leaflet、data-df-map 或地图瓦片;请使用本地 GeoJSON 的 data-df-geo 契约。")
|
||||
if re.search(r"(?:window\.)?echarts\.init\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 ECharts 地图;请只提供 data-df-geo JSON 数据块。")
|
||||
errors.extend(_geo_map_errors(html))
|
||||
return errors
|
||||
|
||||
|
||||
def _quality_report(
|
||||
html: str,
|
||||
*,
|
||||
data_mode: str,
|
||||
expected_features: set[str],
|
||||
expected_page_count: int | None = None,
|
||||
expected_layout_mode: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
lowered = html.lower()
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
required_visualizations = _required_visualizations(expected_features)
|
||||
page_views = _count_page_views(html)
|
||||
if "<!doctype html" not in lowered or "</html>" not in lowered:
|
||||
errors.append("HTML 不完整:需要 <!doctype html> 和 </html>。")
|
||||
if "<body" not in lowered or "</body>" not in lowered:
|
||||
@@ -204,16 +913,41 @@ def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -
|
||||
marker = chart_markers.get(feature)
|
||||
if marker and marker not in lowered:
|
||||
warnings.append(f"未检测到请求的 {feature} 图表结构,请确认是否应删去该要求或补充图表。")
|
||||
font_sizes = [float(value) for value in _FONT_SIZE_RE.findall(html)]
|
||||
too_small = sorted({value for value in font_sizes if value < 16})
|
||||
if too_small:
|
||||
errors.append("发现小于 16px 的字号:" + "、".join(f"{value:g}px" for value in too_small[:8]) + "。")
|
||||
for kind in sorted(required_visualizations):
|
||||
_append_missing_contract_errors(errors, lowered, kind)
|
||||
if "flowchart" in required_visualizations:
|
||||
if re.search(r"class=[\"'][^\"']*\bmermaid\b", lowered) or "mermaid.min.js" in lowered:
|
||||
errors.append("HTML 流程图禁止使用 Mermaid;请转换为 data-df-flow 并由平台 G6 运行时渲染。")
|
||||
if re.search(r"new\s+(?:window\.)?g6\.graph\s*\(", lowered):
|
||||
errors.append("不要在 HTML 中自行初始化 G6.Graph;请只提供 data-df-flow JSON 数据块。")
|
||||
errors.extend(_flow_layout_errors(html))
|
||||
if "chart" in required_visualizations:
|
||||
errors.extend(_chart_errors(html))
|
||||
errors.extend(_dark_component_contrast_errors(html))
|
||||
if "map" in required_visualizations:
|
||||
errors.extend(_map_contract_errors(html))
|
||||
if expected_layout_mode == "document":
|
||||
if re.search(r"(?:html\s*,\s*body|body|html)\s*\{[^}]*\boverflow\s*:\s*hidden", lowered):
|
||||
errors.append("普通单页报告必须允许纵向滚动,禁止对 html/body 设置 overflow:hidden。")
|
||||
fixed_canvas = "1920px" in lowered and "1080px" in lowered
|
||||
viewport_scaler = bool(
|
||||
re.search(r"inner(?:width|height)\s*/\s*(?:1920|1080)", lowered)
|
||||
or re.search(r"scale\s*\([^)]*(?:1920|1080)", lowered)
|
||||
)
|
||||
if fixed_canvas or viewport_scaler:
|
||||
errors.append("普通单页报告禁止固定 1920×1080 舞台和整页等比缩放;请改为响应式文档布局。")
|
||||
if (data_mode or "provided").strip().lower() in {"illustrative", "mixed"} and not any(
|
||||
mark in html for mark in ("示意数据", "待补充数据", "模拟数据")
|
||||
):
|
||||
errors.append("示意图表没有标注“示意数据 / 待补充数据”,可能被误认为事实。")
|
||||
if "reportdata" in lowered and "const reportdata" not in lowered:
|
||||
warnings.append("检测到 reportData 引用但未检测到 const reportData 定义,请检查图表脚本。")
|
||||
if expected_page_count == 1 and page_views > 1:
|
||||
errors.append(f"用户未明确要求多页面,HTML 必须保持单页面;当前检测到 {page_views} 个页面视图。")
|
||||
if expected_page_count is not None and expected_page_count > 1 and page_views < 2:
|
||||
errors.append("用户明确要求多页面,但 HTML 未检测到多个页面视图。")
|
||||
if expected_page_count is not None and expected_page_count > 1:
|
||||
errors.extend(_multi_page_portal_errors(html, page_views))
|
||||
return {
|
||||
"ok": not errors,
|
||||
"errors": errors,
|
||||
@@ -221,27 +955,74 @@ def _quality_report(html: str, *, data_mode: str, expected_features: set[str]) -
|
||||
"metrics": {
|
||||
"bytes": len(html.encode("utf-8")),
|
||||
"text_characters": _count_textual_content(html),
|
||||
"page_sections": len(re.findall(r"class=[\"'][^\"']*(?:report-page|page)[^\"']*[\"']", html, flags=re.IGNORECASE)),
|
||||
"page_sections": page_views,
|
||||
"has_script": "<script" in lowered,
|
||||
"required_visualizations": sorted(required_visualizations),
|
||||
"layout_mode": expected_layout_mode,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _attach_repair_plan(report: dict[str, object], auto_fixed: list[str] | None = None) -> dict[str, object]:
|
||||
"""Turn remaining errors into a one-pass checklist the model must finish together."""
|
||||
notes = [item for item in (auto_fixed or []) if item]
|
||||
errors = [str(item) for item in (report.get("errors") or [])]
|
||||
report["auto_fixed"] = notes
|
||||
report["error_count"] = len(errors)
|
||||
report["repair_checklist"] = [f"{index}. {error}" for index, error in enumerate(errors, start=1)]
|
||||
if report.get("ok"):
|
||||
if notes:
|
||||
report["repair_strategy"] = (
|
||||
"校验已自动修复:" + ";".join(notes) + "。无需再改文件,直接 present_files 交付。"
|
||||
)
|
||||
else:
|
||||
report["repair_strategy"] = "校验通过:无需重写文件,直接 present_files 交付。"
|
||||
return report
|
||||
numbered = "\n".join(str(item) for item in report["repair_checklist"])
|
||||
report["repair_strategy"] = (
|
||||
f"共 {len(errors)} 个错误,必须在同一次回复里全部修完,然后只再 validate 一次。"
|
||||
"禁止修一个就立刻 validate,禁止逐条循环「改一点→再校验」。"
|
||||
"保留当前 HTML;只用最少次数的 str_replace 替换最小的 CSS、导航、JSON 或 DOM 区块。"
|
||||
"禁止重新调用 catalog/prepare,禁止用 write_file 覆盖整个文件。\n"
|
||||
f"{numbered}"
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def _validate(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
*,
|
||||
file_path: str,
|
||||
data_mode: str,
|
||||
expected_features: list[str],
|
||||
preferred_page_count: int | None,
|
||||
page_brief: str,
|
||||
) -> str:
|
||||
actual, thread_id, user_id = _resolve_current_thread_file(runtime, file_path)
|
||||
if actual.stat().st_size > _MAX_HTML_BYTES:
|
||||
raise ValueError(f"HTML 超过 {_MAX_HTML_BYTES // 1024 // 1024}MB,拒绝读取。")
|
||||
html = actual.read_text(encoding="utf-8")
|
||||
report = _quality_report(
|
||||
html,
|
||||
data_mode=data_mode,
|
||||
expected_features={item.strip().lower() for item in expected_features if item.strip()},
|
||||
html, auto_fixed = apply_mechanical_html_fixes(str(actual), html)
|
||||
if auto_fixed:
|
||||
actual.write_text(html, encoding="utf-8")
|
||||
workspace = get_paths().sandbox_work_dir(thread_id, user_id=user_id)
|
||||
persisted_visualizations = _brief_visualizations(actual, workspace)
|
||||
expected_page_count = _brief_page_count(actual, workspace)
|
||||
if expected_page_count is None:
|
||||
expected_page_count = _requested_page_count(preferred_page_count, page_brief)
|
||||
expected_layout_mode = _brief_layout_mode(actual, workspace) or _requested_layout_mode(page_brief)
|
||||
merged_features = {item.strip().lower() for item in expected_features if item.strip()}
|
||||
merged_features.update(persisted_visualizations)
|
||||
merged_features.update(_required_visualizations([], page_brief))
|
||||
report = _attach_repair_plan(
|
||||
_quality_report(
|
||||
html,
|
||||
data_mode=data_mode,
|
||||
expected_features=merged_features,
|
||||
expected_page_count=expected_page_count,
|
||||
expected_layout_mode=expected_layout_mode,
|
||||
),
|
||||
auto_fixed,
|
||||
)
|
||||
report_path = actual.with_name(f"{actual.stem}.quality.json")
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
@@ -268,26 +1049,41 @@ def page_report_template_tool(
|
||||
) -> str:
|
||||
"""Select, copy, and quality-check reusable HTML report-page templates.
|
||||
|
||||
Only use this tool when the current user request explicitly asks for an HTML
|
||||
file, webpage, page, dashboard, or large-screen view. A request to draw a
|
||||
flowchart, chart, map, or visualization alone is not permission to use it.
|
||||
HTML is generated from scratch by default: write it directly under
|
||||
``/mnt/user-data/outputs/`` and use only ``validate``. Use ``catalog`` and
|
||||
``prepare`` only when the user explicitly requests a template. Always
|
||||
present the completed artifact; never substitute an inline HTML code block.
|
||||
|
||||
Use this tool instead of editing a source template in place.
|
||||
|
||||
Args:
|
||||
action: One of ``catalog``, ``prepare``, or ``validate``. ``catalog`` lists
|
||||
profiles; ``prepare`` selects/copies a source into this thread's
|
||||
workspace and writes an editing brief; ``validate`` checks an edited
|
||||
HTML file and writes a JSON quality report next to it.
|
||||
profiles; ``prepare`` is opt-in and selects/copies a user-requested
|
||||
template into this thread's outputs; ``validate`` also supports
|
||||
from-scratch HTML and writes a JSON quality report next to it.
|
||||
theme: ``dark`` or ``light``. Use the UI-provided presentation theme when
|
||||
available; ``auto`` lets the selector use all profiles.
|
||||
content_density: ``text-heavy``, ``balanced``, or ``data-heavy``.
|
||||
required_features: Needed components such as ``line``, ``pie``,
|
||||
``radar``, ``scatter``, ``gantt``, ``table``, ``text-panels``.
|
||||
required_features: Needed components such as ``flowchart``, ``chart``,
|
||||
``map``, ``line``, ``pie``, ``radar``, ``scatter``, ``gantt``,
|
||||
``table``, or ``text-panels``. Preserve source Markdown Mermaid as
|
||||
``flowchart`` for both ``prepare`` and ``validate``.
|
||||
preferred_page_count: Preferred number of report pages. The agent may
|
||||
remove unused skeleton pages after preparation.
|
||||
use a value greater than one only when the current user explicitly
|
||||
asks for multiple pages or specifies a page count. Omitted defaults
|
||||
to one page; pass it to validate even when prepare was skipped.
|
||||
selection_key: Stable request/thread-derived text used to vary equally
|
||||
good template choices.
|
||||
exclude_template_ids: Recently-used template ids to avoid where possible.
|
||||
template_id: Optional explicit catalog id. Omit to select automatically.
|
||||
output_name: Final safe HTML filename, e.g. ``经营月报.html``.
|
||||
page_brief: Concise summary of supplied materials and desired output.
|
||||
Mention every source flowchart, chart, and map; prepare persists
|
||||
these requirements, while direct validation reads them from this
|
||||
argument. Always pass it to validate when prepare was skipped.
|
||||
file_path: For ``validate`` only: current-thread HTML under workspace or
|
||||
outputs.
|
||||
data_mode: ``provided`` for factual data, ``illustrative`` when charts
|
||||
@@ -315,6 +1111,8 @@ def page_report_template_tool(
|
||||
file_path=file_path,
|
||||
data_mode=data_mode,
|
||||
expected_features=required_features or [],
|
||||
preferred_page_count=preferred_page_count,
|
||||
page_brief=page_brief,
|
||||
)
|
||||
raise ValueError("action 仅支持 catalog、prepare、validate。")
|
||||
|
||||
|
||||
+109
-23
@@ -4,19 +4,77 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import PurePosixPath
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.skills.storage import SkillStorage
|
||||
from deerflow.skills.types import Skill
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ROUNDTABLE_COORDINATOR_AGENT_ID = "roundtable-coordinator"
|
||||
COORDINATOR_AGENT_IDS = frozenset(
|
||||
{_ROUNDTABLE_COORDINATOR_AGENT_ID, "task-analysis-coordinator"}
|
||||
)
|
||||
COORDINATOR_ONLY_SKILL_NAMES = frozenset({"agent_orchestration", "agent-orchestration"})
|
||||
|
||||
|
||||
class SkillPathUnavailableError(PermissionError):
|
||||
"""Raised when a model tool targets an unknown or offline skill path."""
|
||||
|
||||
|
||||
def require_enabled_skill_path(
|
||||
path: str,
|
||||
*,
|
||||
container_path: str = "/mnt/skills",
|
||||
storage: SkillStorage | None = None,
|
||||
) -> Skill:
|
||||
"""Return the discovered enabled skill containing ``path`` or deny access.
|
||||
|
||||
Matching uses the discovered directory instead of assuming it equals the
|
||||
frontmatter name. Root/category-wide paths and unregistered files fail
|
||||
closed; models discover skills through ``skill_list``/``search_skills``.
|
||||
"""
|
||||
normalized_container = container_path.replace("\\", "/").rstrip("/") or "/"
|
||||
normalized_path = path.replace("\\", "/").rstrip("/") or "/"
|
||||
if normalized_path != normalized_container and not normalized_path.startswith(f"{normalized_container}/"):
|
||||
raise SkillPathUnavailableError("Path is outside the skills mount")
|
||||
if ".." in PurePosixPath(normalized_path).parts:
|
||||
raise SkillPathUnavailableError("Skill path traversal is not allowed")
|
||||
|
||||
try:
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
# SkillStorage intentionally degrades open on extensions-config errors;
|
||||
# model-facing file access must instead fail closed.
|
||||
extensions = ExtensionsConfig.from_file()
|
||||
catalog = (storage or get_or_new_skill_storage()).load_skills(enabled_only=False)
|
||||
except Exception as exc:
|
||||
raise SkillPathUnavailableError("Skill availability could not be verified") from exc
|
||||
|
||||
matches: list[tuple[int, Skill]] = []
|
||||
for skill in catalog:
|
||||
skill_root = skill.get_container_path(normalized_container).replace("\\", "/").rstrip("/")
|
||||
if normalized_path == skill_root or normalized_path.startswith(f"{skill_root}/"):
|
||||
matches.append((len(skill_root), skill))
|
||||
if not matches:
|
||||
raise SkillPathUnavailableError("Skill path is unavailable")
|
||||
|
||||
skill = max(matches, key=lambda item: item[0])[1]
|
||||
category = skill.category.value if hasattr(skill.category, "value") else str(skill.category)
|
||||
if not extensions.is_skill_enabled(skill.name, category):
|
||||
raise SkillPathUnavailableError("Skill path is unavailable")
|
||||
return skill
|
||||
|
||||
|
||||
def can_access_coordinator_only_skill(agent_id: str | None, skill_name: str) -> bool:
|
||||
"""Return whether this runtime identity may discover/read a reserved orchestration skill."""
|
||||
normalized = (skill_name or "").strip().lower()
|
||||
return normalized not in COORDINATOR_ONLY_SKILL_NAMES or agent_id == _ROUNDTABLE_COORDINATOR_AGENT_ID
|
||||
return normalized not in COORDINATOR_ONLY_SKILL_NAMES or agent_id in COORDINATOR_AGENT_IDS
|
||||
|
||||
|
||||
def is_coordinator_only_skill_path(path: str) -> bool:
|
||||
@@ -66,22 +124,31 @@ def _load_agent_skill_whitelist(agent_id: str | None, extra_skills: list[str] |
|
||||
through this tool.
|
||||
- Non-empty set → only these names are visible to this agent.
|
||||
|
||||
Exception — the deep-research collector (``deep-research-collector``) has no
|
||||
whitelist: it sees every skill like the default lead agent, matching the
|
||||
runtime allowlist in ``lead_agent.agent``. Its on-disk ``skills:`` entry is
|
||||
ignored.
|
||||
|
||||
``extra_skills`` only widens the **deep-research collector** allowlist
|
||||
for one run (report-structure 检索来源). Other agents ignore it.
|
||||
"""
|
||||
if not agent_id:
|
||||
return None
|
||||
try:
|
||||
from deerflow.agents.deep_research.retrieval import COLLECTOR_AGENT_ID
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
|
||||
cfg = load_agent_config(agent_id)
|
||||
if cfg is None:
|
||||
return None
|
||||
allowed = {
|
||||
name
|
||||
for name in (cfg.skills or [])
|
||||
if can_access_coordinator_only_skill(agent_id, name)
|
||||
}
|
||||
if agent_id == COLLECTOR_AGENT_ID:
|
||||
allowed = None # collector: all skills, like the default lead agent
|
||||
else:
|
||||
allowed = {
|
||||
name
|
||||
for name in (cfg.skills or [])
|
||||
if can_access_coordinator_only_skill(agent_id, name)
|
||||
}
|
||||
except Exception:
|
||||
logger.debug("Failed to load agent config for skill_list filter", exc_info=True)
|
||||
return None
|
||||
@@ -89,8 +156,8 @@ def _load_agent_skill_whitelist(agent_id: str | None, extra_skills: list[str] |
|
||||
if extra_skills:
|
||||
from deerflow.agents.deep_research.retrieval import merge_collector_skill_allowlist
|
||||
|
||||
merged = merge_collector_skill_allowlist(agent_id, list(allowed), extra_skills)
|
||||
return set(merged or [])
|
||||
merged = merge_collector_skill_allowlist(agent_id, None if allowed is None else list(allowed), extra_skills)
|
||||
return set(merged or []) if merged is not None else None
|
||||
return allowed
|
||||
|
||||
|
||||
@@ -169,20 +236,21 @@ def skill_view_impl(name: str, agent_id: str | None, extra_skills: list[str] | N
|
||||
|
||||
storage = get_or_new_skill_storage()
|
||||
try:
|
||||
if storage.custom_skill_exists(name):
|
||||
content = storage.read_custom_skill(name)
|
||||
elif storage.public_skill_exists(name):
|
||||
skill_file = storage.get_skills_root_path()
|
||||
from pathlib import Path
|
||||
# A warm catalogue already knows each skill's file. Read that one file.
|
||||
# A cold catalogue checks the canonical directory before walking every
|
||||
# SKILL.md, which is only needed for a directory whose name differs.
|
||||
from deerflow.agents.lead_agent.prompt import _load_named_enabled_skill, get_warm_enabled_skills
|
||||
|
||||
from deerflow.skills.types import SKILL_MD_FILE
|
||||
|
||||
path = Path(skill_file) / "public" / name / SKILL_MD_FILE
|
||||
if not path.exists():
|
||||
return f"Skill '{name}' not found."
|
||||
content = path.read_text(encoding="utf-8")
|
||||
cached = get_warm_enabled_skills(storage)
|
||||
if cached is not None:
|
||||
skill = next((item for item in cached if item.name == name), None)
|
||||
else:
|
||||
skill = _load_named_enabled_skill(name)
|
||||
if skill is None:
|
||||
skill = next((item for item in storage.load_skills(enabled_only=True) if item.name == name), None)
|
||||
if skill is None:
|
||||
return f"Skill '{name}' not found."
|
||||
content = skill.skill_file.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
logger.debug("skill_view failed for %r: %s", name, e)
|
||||
return f"Error reading skill '{name}': {e}"
|
||||
@@ -260,7 +328,7 @@ def search_skills_impl(query: str, agent_id: str | None, top_n: int = 5, extra_s
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
skills = _skills_for_catalogue(storage)
|
||||
usage = all_entries()
|
||||
base_path = _resolve_skills_container_base_path()
|
||||
scored: list[tuple[int, dict]] = []
|
||||
@@ -336,11 +404,29 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
return skill_list_impl(_resolve_active_agent_id(runtime), _resolve_extra_skills(runtime))
|
||||
|
||||
|
||||
def _latest_activity(entry: dict) -> str | None:
|
||||
candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
|
||||
valid = [ts for ts in candidates if ts]
|
||||
if valid:
|
||||
return max(valid)
|
||||
return entry.get("created_at")
|
||||
|
||||
|
||||
def _skills_for_catalogue(storage):
|
||||
"""Use the warm prompt catalogue, and parse SKILL.md files only when it is cold."""
|
||||
from deerflow.agents.lead_agent.prompt import get_warm_enabled_skills
|
||||
|
||||
cached = get_warm_enabled_skills(storage)
|
||||
if cached is not None:
|
||||
return cached
|
||||
return storage.load_skills(enabled_only=True)
|
||||
|
||||
|
||||
def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None) -> str:
|
||||
"""List skills visible to ``agent_id`` (None = default lead agent)."""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries
|
||||
|
||||
allowed = _load_agent_skill_whitelist(agent_id, extra_skills)
|
||||
|
||||
@@ -353,7 +439,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
|
||||
|
||||
try:
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
skills = _skills_for_catalogue(storage)
|
||||
usage = all_entries()
|
||||
result = []
|
||||
for skill in skills:
|
||||
@@ -380,7 +466,7 @@ def skill_list_impl(agent_id: str | None, extra_skills: list[str] | None = None)
|
||||
"use_count": entry.get("use_count", 0),
|
||||
"view_count": entry.get("view_count", 0),
|
||||
"patch_count": entry.get("patch_count", 0),
|
||||
"last_activity_at": latest_activity_at(skill.name),
|
||||
"last_activity_at": _latest_activity(entry),
|
||||
"pinned": entry.get("pinned", False),
|
||||
"source": entry.get("source", "upload"),
|
||||
})
|
||||
|
||||
+62
-11
@@ -1,5 +1,6 @@
|
||||
import base64
|
||||
import mimetypes
|
||||
import base64
|
||||
import io
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
@@ -17,7 +18,10 @@ _ALLOWED_IMAGE_VIRTUAL_ROOTS = (
|
||||
f"{VIRTUAL_PATH_PREFIX}/outputs",
|
||||
)
|
||||
_ALLOWED_IMAGE_VIRTUAL_ROOTS_TEXT = ", ".join(_ALLOWED_IMAGE_VIRTUAL_ROOTS)
|
||||
_MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
||||
_MAX_IMAGE_BYTES = 100 * 1024 * 1024
|
||||
_MAX_IMAGE_PIXELS = 40_000_000
|
||||
_MAX_ANALYSIS_SIDE = 2048
|
||||
_MAX_ANALYSIS_BYTES = 4 * 1024 * 1024
|
||||
_EXTENSION_TO_MIME = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
@@ -40,10 +44,44 @@ def _detect_image_mime(image_data: bytes) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _sanitize_image_error(error: Exception, thread_data: ThreadDataState | None) -> str:
|
||||
def _sanitize_image_error(error: Exception, thread_data: ThreadDataState | None) -> str:
|
||||
from deerflow.sandbox.tools import mask_local_paths_in_output
|
||||
|
||||
return mask_local_paths_in_output(f"{type(error).__name__}: {error}", thread_data)
|
||||
return mask_local_paths_in_output(f"{type(error).__name__}: {error}", thread_data)
|
||||
|
||||
|
||||
def _bounded_analysis_image(image_data: bytes, mime_type: str) -> tuple[bytes, str]:
|
||||
"""Validate decoded dimensions and create a bounded vision derivative."""
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Pillow is required for safe image dimension validation") from exc
|
||||
|
||||
with Image.open(io.BytesIO(image_data)) as image:
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0 or width * height > _MAX_IMAGE_PIXELS:
|
||||
raise ValueError(f"decoded image dimensions are unsafe: {width}x{height}")
|
||||
if max(width, height) <= _MAX_ANALYSIS_SIDE and len(image_data) <= _MAX_ANALYSIS_BYTES:
|
||||
image.verify()
|
||||
return image_data, mime_type
|
||||
|
||||
with Image.open(io.BytesIO(image_data)) as image:
|
||||
image.thumbnail((_MAX_ANALYSIS_SIDE, _MAX_ANALYSIS_SIDE))
|
||||
output = io.BytesIO()
|
||||
if mime_type == "image/jpeg":
|
||||
image.convert("RGB").save(output, format="JPEG", quality=85, optimize=True)
|
||||
output_mime = "image/jpeg"
|
||||
else:
|
||||
image.save(output, format="PNG", optimize=True)
|
||||
output_mime = "image/png"
|
||||
bounded = output.getvalue()
|
||||
if len(bounded) > _MAX_ANALYSIS_BYTES:
|
||||
output = io.BytesIO()
|
||||
image.thumbnail((1536, 1536))
|
||||
image.convert("RGB").save(output, format="JPEG", quality=82, optimize=True)
|
||||
bounded = output.getvalue()
|
||||
output_mime = "image/jpeg"
|
||||
return bounded, output_mime
|
||||
|
||||
|
||||
@tool("view_image", parse_docstring=True)
|
||||
@@ -140,9 +178,16 @@ def view_image_tool(
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error reading image metadata: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
if image_size > _MAX_IMAGE_BYTES:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error: Image file is too large: {image_size} bytes. Maximum supported size is {_MAX_IMAGE_BYTES} bytes", tool_call_id=tool_call_id)]},
|
||||
try:
|
||||
from deerflow.uploads.policy import get_runtime_upload_policy
|
||||
|
||||
image_limit = int(((get_runtime_upload_policy().get("groups") or {}).get("image") or {}).get("max_bytes") or _MAX_IMAGE_BYTES)
|
||||
except Exception:
|
||||
image_limit = _MAX_IMAGE_BYTES
|
||||
image_limit = min(max(1, image_limit), _MAX_IMAGE_BYTES)
|
||||
if image_size > image_limit:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error: Image file is too large: {image_size} bytes. Maximum supported size is {image_limit} bytes", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
|
||||
# Read image file and convert to base64
|
||||
@@ -159,12 +204,18 @@ def view_image_tool(
|
||||
return Command(
|
||||
update={"messages": [ToolMessage("Error: File contents do not match a supported image format", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
if detected_mime_type != expected_mime_type:
|
||||
if detected_mime_type != expected_mime_type:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error: Image contents are {detected_mime_type}, but file extension indicates {expected_mime_type}", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
mime_type = detected_mime_type
|
||||
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
||||
mime_type = detected_mime_type
|
||||
try:
|
||||
image_data, mime_type = _bounded_analysis_image(image_data, mime_type)
|
||||
except Exception as e:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error validating image dimensions: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
||||
|
||||
# Delegated recognition: when a dedicated vision model is configured, send
|
||||
# the image to that model and return its description as the tool result, so
|
||||
|
||||
Reference in new issue
Block a user