fix: 写作管线整批接收对话检索结果,不再自己检索

- 相关性过滤新增位置通道:收割器给每条材料标记来源消息位置,课题
  请求之后的检索结果整批入池(与语言/词面无关)——线上中文课题 +
  英文检索结果被纯词面匹配误杀 31/35 条,报告只带着 4 条残料在写
- 课题切换保护保留:新请求之前的旧课题检索仍按词面强过滤,不凑
  门槛、不入报告;对话里找不到课题消息时退化为原词面过滤
- 删除死代码 HarvestPlusSearchMaterialProvider(retrieve=True 的
  自检索补充分支,早已被显式忽略):写作管线只负责写文件,检索
  全部由对话完成成为结构保证
- 测试:锚点规则(澄清回答不充当新课题)、中文课题+英文资料整批
  保留、课题切换旧检索过滤等场景,25 个用例入库(此前该测试文件
  一直被 tests/* 忽略规则挡在版本库外)
This commit is contained in:
董宏凯 committed 2026-08-30 16:20:55 +08:00
1 parent f7e6ed65bd
commit 2ab3ab8444
4 files changed
+757 -73

No files matched your search

@@ -34,7 +34,7 @@ import re
from typing import Annotated, Any
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
from langchain_core.messages import ToolMessage
from langchain_core.messages import HumanMessage, ToolMessage
from langgraph.types import Command
from langgraph.typing import ContextT
@@ -42,7 +42,6 @@ from deerflow.agents.deep_research.adapters.artifacts import ThreadOutputsArtifa
from deerflow.agents.deep_research.adapters.context import DeerFlowContextCompressor
from deerflow.agents.deep_research.adapters.image import build_image_provider
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
from deerflow.agents.deep_research.adapters.material_provider import DeerFlowMaterialProvider
from deerflow.agents.deep_research.collection.thread_harvester import (
StaticHarvestMaterialProvider,
harvest_messages_materials,
@@ -167,53 +166,6 @@ class StreamWriterEventSink:
logger.debug("deep_research_report: custom stream frame dropped", exc_info=True)
class HarvestPlusSearchMaterialProvider:
"""对话收割池优先 + 自检索补充的合并资料源(``retrieve=True`` 分支)。
与 :class:`StaticHarvestMaterialProvider` 同协议:每个子查询返回合并池,
交由 runner 的 dedupe / compress 阶段裁剪。自检索单项失败不致命——
退化为只用对话资料。
"""
def __init__(self, harvested: list[ResearchMaterial], search_provider: DeerFlowMaterialProvider) -> None:
self._harvested = list(harvested)
self._search = search_provider
@property
def material_count(self) -> int:
return len(self._harvested)
async def search(
self,
query: str,
*,
domains: list[str] | None = None,
limit: int = 8,
user_id: str = "",
session_id: str = "",
cancellation=None,
) -> list[ResearchMaterial]:
from deerflow.agents.deep_research.adapters.material import dedupe_materials
if cancellation is not None:
cancellation.raise_if_cancelled()
combined = list(self._harvested)
try:
combined.extend(
await self._search.search(
query,
domains=domains,
limit=limit,
user_id=user_id,
session_id=session_id,
cancellation=cancellation,
)
)
except Exception: # noqa: BLE001 - 自检索失败退化为对话资料
logger.warning("deep_research_report: supplementary search failed", exc_info=True)
return dedupe_materials(combined)
# 写作模式常在「不用搜索 / 对话资料很少」时开写。工作台默认的「材料不足就
# 写明资料不足」会把报告写成元分析;这里强制用领域知识补全正文。
_KNOWLEDGE_WRITE_INSTRUCTION = (
@@ -330,16 +282,91 @@ def _is_topic_relevant(material: ResearchMaterial, strong: frozenset[str], weak:
def _filter_relevant_materials(
materials: list[ResearchMaterial], *, topic: str
materials: list[ResearchMaterial],
*,
topic: str,
anchor_index: int | None = None,
) -> tuple[list[ResearchMaterial], int]:
"""按当前主题过滤收割池;返回 ``(相关池, 被剔除条数)``。"""
"""按当前主题过滤收割池;返回 ``(相关池, 被剔除条数)``。
两条保留通道(满足其一即入池):
1. **位置**:材料来自课题请求消息(``anchor_index``)之后的工具消息——
那是模型为本课题检索的,与词面/语言无关,一律保留。线上实测中文课题
+ 英文检索结果会被纯词面匹配整批误杀(35 条只留 4 条)。
2. **词面**:材料词面命中课题(强词/标题/正文规则)——兜住课题切换前
的旧检索里仍与当前课题相关的部分。
``anchor_index`` 之前的旧课题检索(词面不命中)继续被剔除(P1:旧课题
资料不得凑门槛、不得入报告)。``anchor_index=None`` 或负数(对话里找不
到课题请求消息,如纯工具消息的旧调用方)退化为纯词面过滤。
"""
strong, weak = _topic_terms(topic)
if not strong and not weak:
return list(materials), 0
relevant = [m for m in materials if _is_topic_relevant(m, strong, weak)]
positional = anchor_index is not None and anchor_index >= 0
relevant: list[ResearchMaterial] = []
for material in materials:
message_index = material.metadata.get("message_index")
if positional and isinstance(message_index, int) and message_index > anchor_index:
relevant.append(material)
continue
if _is_topic_relevant(material, strong, weak):
relevant.append(material)
return relevant, len(materials) - len(relevant)
def _anchor_visible_human(message: Any) -> bool:
if not isinstance(message, HumanMessage):
return False
kwargs = getattr(message, "additional_kwargs", None) or {}
return not kwargs.get("hide_from_ui")
def _anchor_is_clarification_result(message: Any) -> bool:
return isinstance(message, ToolMessage) and str(getattr(message, "name", "") or "") == "ask_clarification"
def _anchor_message_text(message: Any) -> str:
content = getattr(message, "content", "")
if isinstance(content, list):
parts: list[str] = []
for part in content:
if isinstance(part, str):
parts.append(part)
elif isinstance(part, dict) and isinstance(part.get("text"), str):
parts.append(part["text"])
return " ".join(parts).strip()
return str(content or "").strip()
def topic_anchor_index(messages: list[Any], *, topic: str = "") -> int:
"""课题请求消息在对话中的位置(写作模式首报轮的资料过滤锚点)。
规则与 ``writing_mode_report_middleware.extract_report_topic`` 一致:取
最新的可见人类消息、跳过对 ``ask_clarification`` 的直接回答(紧跟澄清
ToolMessage 的人类消息不是新课题)。传入 ``topic`` 时优先返回与其文本
前缀一致的消息位置(模型原样复述课题的常见情形)。找不到返回 -1。
"""
normalized_topic = " ".join(str(topic or "").split())
prefix = normalized_topic[:80]
fallback = -1
for index in range(len(messages) - 1, -1, -1):
message = messages[index]
if not _anchor_visible_human(message):
continue
if index > 0 and _anchor_is_clarification_result(messages[index - 1]):
continue
text = " ".join(_anchor_message_text(message).split())
if not text:
continue
if prefix and text.startswith(prefix):
return index
if fallback < 0:
fallback = index
return fallback
def _safe_stream_writer() -> Any:
"""获取当前 run 的 custom 事件写入器;非流式上下文返回 None。"""
try:
@@ -408,16 +435,18 @@ async def deep_research_report_tool(
state = getattr(runtime, "state", None) or {}
conversation_messages = state.get("messages") or []
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query)
# 按当前主题强过滤:旧课题的检索结果不计门槛、不入报告(课题切换场景)。
materials, dropped_count = _filter_relevant_materials(harvested, topic=query)
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
anchor_index = topic_anchor_index(conversation_messages, topic=query)
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
harvested_count = len(materials)
logger.info(
"[deep_research_report] thread=%s harvested %d materials from conversation "
"(topic-relevant %d, off-topic dropped %d)",
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
thread_id,
len(harvested),
harvested_count,
dropped_count,
anchor_index,
)
if harvested_count < _MIN_HARVESTED_MATERIALS:
@@ -430,8 +459,7 @@ async def deep_research_report_tool(
if retrieve is True:
logger.info(
"[deep_research_report] thread=%s ignoring retrieve=true; "
"writing-mode pipeline writes from conversation materials only",
"[deep_research_report] thread=%s ignoring retrieve=true; writing-mode pipeline writes from conversation materials only",
thread_id,
)
materials_provider: Any = StaticHarvestMaterialProvider(materials)
@@ -520,11 +548,7 @@ async def deep_research_report_tool(
)
source_count = len(result.source_ids or [])
summary = (
f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,"
f"已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。"
"请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
)
summary = f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
logger.info(
"[deep_research_report] thread=%s completed (sources=%d, report_chars=%d)",
thread_id,
@@ -540,10 +564,10 @@ async def deep_research_report_tool(
__all__ = [
"HarvestPlusSearchMaterialProvider",
"REPORT_VIRTUAL_PATH",
"StreamWriterEventSink",
"build_writing_research_config",
"deep_research_report_tool",
"is_writing_mode_report_path",
"topic_anchor_index",
]