fix: 写作管线整批接收对话检索结果,不再自己检索
- 相关性过滤新增位置通道:收割器给每条材料标记来源消息位置,课题 请求之后的检索结果整批入池(与语言/词面无关)——线上中文课题 + 英文检索结果被纯词面匹配误杀 31/35 条,报告只带着 4 条残料在写 - 课题切换保护保留:新请求之前的旧课题检索仍按词面强过滤,不凑 门槛、不入报告;对话里找不到课题消息时退化为原词面过滤 - 删除死代码 HarvestPlusSearchMaterialProvider(retrieve=True 的 自检索补充分支,早已被显式忽略):写作管线只负责写文件,检索 全部由对话完成成为结构保证 - 测试:锚点规则(澄清回答不充当新课题)、中文课题+英文资料整批 保留、课题切换旧检索过滤等场景,25 个用例入库(此前该测试文件 一直被 tests/* 忽略规则挡在版本库外)
This commit is contained in:
1 parent
f7e6ed65bd
commit
2ab3ab8444
4 files changed
+757
-73
No files matched your search
+88
-64
@@ -34,7 +34,7 @@ import re
|
||||
from typing import Annotated, Any
|
||||
|
||||
from langchain.tools import InjectedToolCallId, ToolRuntime, tool
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
from langgraph.typing import ContextT
|
||||
|
||||
@@ -42,7 +42,6 @@ from deerflow.agents.deep_research.adapters.artifacts import ThreadOutputsArtifa
|
||||
from deerflow.agents.deep_research.adapters.context import DeerFlowContextCompressor
|
||||
from deerflow.agents.deep_research.adapters.image import build_image_provider
|
||||
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
|
||||
from deerflow.agents.deep_research.adapters.material_provider import DeerFlowMaterialProvider
|
||||
from deerflow.agents.deep_research.collection.thread_harvester import (
|
||||
StaticHarvestMaterialProvider,
|
||||
harvest_messages_materials,
|
||||
@@ -167,53 +166,6 @@ class StreamWriterEventSink:
|
||||
logger.debug("deep_research_report: custom stream frame dropped", exc_info=True)
|
||||
|
||||
|
||||
class HarvestPlusSearchMaterialProvider:
|
||||
"""对话收割池优先 + 自检索补充的合并资料源(``retrieve=True`` 分支)。
|
||||
|
||||
与 :class:`StaticHarvestMaterialProvider` 同协议:每个子查询返回合并池,
|
||||
交由 runner 的 dedupe / compress 阶段裁剪。自检索单项失败不致命——
|
||||
退化为只用对话资料。
|
||||
"""
|
||||
|
||||
def __init__(self, harvested: list[ResearchMaterial], search_provider: DeerFlowMaterialProvider) -> None:
|
||||
self._harvested = list(harvested)
|
||||
self._search = search_provider
|
||||
|
||||
@property
|
||||
def material_count(self) -> int:
|
||||
return len(self._harvested)
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
domains: list[str] | None = None,
|
||||
limit: int = 8,
|
||||
user_id: str = "",
|
||||
session_id: str = "",
|
||||
cancellation=None,
|
||||
) -> list[ResearchMaterial]:
|
||||
from deerflow.agents.deep_research.adapters.material import dedupe_materials
|
||||
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
combined = list(self._harvested)
|
||||
try:
|
||||
combined.extend(
|
||||
await self._search.search(
|
||||
query,
|
||||
domains=domains,
|
||||
limit=limit,
|
||||
user_id=user_id,
|
||||
session_id=session_id,
|
||||
cancellation=cancellation,
|
||||
)
|
||||
)
|
||||
except Exception: # noqa: BLE001 - 自检索失败退化为对话资料
|
||||
logger.warning("deep_research_report: supplementary search failed", exc_info=True)
|
||||
return dedupe_materials(combined)
|
||||
|
||||
|
||||
# 写作模式常在「不用搜索 / 对话资料很少」时开写。工作台默认的「材料不足就
|
||||
# 写明资料不足」会把报告写成元分析;这里强制用领域知识补全正文。
|
||||
_KNOWLEDGE_WRITE_INSTRUCTION = (
|
||||
@@ -330,16 +282,91 @@ def _is_topic_relevant(material: ResearchMaterial, strong: frozenset[str], weak:
|
||||
|
||||
|
||||
def _filter_relevant_materials(
|
||||
materials: list[ResearchMaterial], *, topic: str
|
||||
materials: list[ResearchMaterial],
|
||||
*,
|
||||
topic: str,
|
||||
anchor_index: int | None = None,
|
||||
) -> tuple[list[ResearchMaterial], int]:
|
||||
"""按当前主题过滤收割池;返回 ``(相关池, 被剔除条数)``。"""
|
||||
"""按当前主题过滤收割池;返回 ``(相关池, 被剔除条数)``。
|
||||
|
||||
两条保留通道(满足其一即入池):
|
||||
|
||||
1. **位置**:材料来自课题请求消息(``anchor_index``)之后的工具消息——
|
||||
那是模型为本课题检索的,与词面/语言无关,一律保留。线上实测中文课题
|
||||
+ 英文检索结果会被纯词面匹配整批误杀(35 条只留 4 条)。
|
||||
2. **词面**:材料词面命中课题(强词/标题/正文规则)——兜住课题切换前
|
||||
的旧检索里仍与当前课题相关的部分。
|
||||
|
||||
``anchor_index`` 之前的旧课题检索(词面不命中)继续被剔除(P1:旧课题
|
||||
资料不得凑门槛、不得入报告)。``anchor_index=None`` 或负数(对话里找不
|
||||
到课题请求消息,如纯工具消息的旧调用方)退化为纯词面过滤。
|
||||
"""
|
||||
strong, weak = _topic_terms(topic)
|
||||
if not strong and not weak:
|
||||
return list(materials), 0
|
||||
relevant = [m for m in materials if _is_topic_relevant(m, strong, weak)]
|
||||
positional = anchor_index is not None and anchor_index >= 0
|
||||
relevant: list[ResearchMaterial] = []
|
||||
for material in materials:
|
||||
message_index = material.metadata.get("message_index")
|
||||
if positional and isinstance(message_index, int) and message_index > anchor_index:
|
||||
relevant.append(material)
|
||||
continue
|
||||
if _is_topic_relevant(material, strong, weak):
|
||||
relevant.append(material)
|
||||
return relevant, len(materials) - len(relevant)
|
||||
|
||||
|
||||
def _anchor_visible_human(message: Any) -> bool:
|
||||
if not isinstance(message, HumanMessage):
|
||||
return False
|
||||
kwargs = getattr(message, "additional_kwargs", None) or {}
|
||||
return not kwargs.get("hide_from_ui")
|
||||
|
||||
|
||||
def _anchor_is_clarification_result(message: Any) -> bool:
|
||||
return isinstance(message, ToolMessage) and str(getattr(message, "name", "") or "") == "ask_clarification"
|
||||
|
||||
|
||||
def _anchor_message_text(message: Any) -> str:
|
||||
content = getattr(message, "content", "")
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
parts.append(part)
|
||||
elif isinstance(part, dict) and isinstance(part.get("text"), str):
|
||||
parts.append(part["text"])
|
||||
return " ".join(parts).strip()
|
||||
return str(content or "").strip()
|
||||
|
||||
|
||||
def topic_anchor_index(messages: list[Any], *, topic: str = "") -> int:
|
||||
"""课题请求消息在对话中的位置(写作模式首报轮的资料过滤锚点)。
|
||||
|
||||
规则与 ``writing_mode_report_middleware.extract_report_topic`` 一致:取
|
||||
最新的可见人类消息、跳过对 ``ask_clarification`` 的直接回答(紧跟澄清
|
||||
ToolMessage 的人类消息不是新课题)。传入 ``topic`` 时优先返回与其文本
|
||||
前缀一致的消息位置(模型原样复述课题的常见情形)。找不到返回 -1。
|
||||
"""
|
||||
normalized_topic = " ".join(str(topic or "").split())
|
||||
prefix = normalized_topic[:80]
|
||||
fallback = -1
|
||||
for index in range(len(messages) - 1, -1, -1):
|
||||
message = messages[index]
|
||||
if not _anchor_visible_human(message):
|
||||
continue
|
||||
if index > 0 and _anchor_is_clarification_result(messages[index - 1]):
|
||||
continue
|
||||
text = " ".join(_anchor_message_text(message).split())
|
||||
if not text:
|
||||
continue
|
||||
if prefix and text.startswith(prefix):
|
||||
return index
|
||||
if fallback < 0:
|
||||
fallback = index
|
||||
return fallback
|
||||
|
||||
|
||||
def _safe_stream_writer() -> Any:
|
||||
"""获取当前 run 的 custom 事件写入器;非流式上下文返回 None。"""
|
||||
try:
|
||||
@@ -408,16 +435,18 @@ async def deep_research_report_tool(
|
||||
state = getattr(runtime, "state", None) or {}
|
||||
conversation_messages = state.get("messages") or []
|
||||
harvested, _agent_summary = harvest_messages_materials(conversation_messages, query=query)
|
||||
# 按当前主题强过滤:旧课题的检索结果不计门槛、不入报告(课题切换场景)。
|
||||
materials, dropped_count = _filter_relevant_materials(harvested, topic=query)
|
||||
# 课题请求之后的检索结果整批保留(那是模型为本课题检索的);只有课题
|
||||
# 切换前的旧检索才按词面强过滤(旧课题资料不得凑门槛、不得入报告)。
|
||||
anchor_index = topic_anchor_index(conversation_messages, topic=query)
|
||||
materials, dropped_count = _filter_relevant_materials(harvested, topic=query, anchor_index=anchor_index)
|
||||
harvested_count = len(materials)
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s harvested %d materials from conversation "
|
||||
"(topic-relevant %d, off-topic dropped %d)",
|
||||
"[deep_research_report] thread=%s harvested %d materials from conversation (topic-relevant %d, off-topic dropped %d, anchor_index=%d)",
|
||||
thread_id,
|
||||
len(harvested),
|
||||
harvested_count,
|
||||
dropped_count,
|
||||
anchor_index,
|
||||
)
|
||||
|
||||
if harvested_count < _MIN_HARVESTED_MATERIALS:
|
||||
@@ -430,8 +459,7 @@ async def deep_research_report_tool(
|
||||
|
||||
if retrieve is True:
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s ignoring retrieve=true; "
|
||||
"writing-mode pipeline writes from conversation materials only",
|
||||
"[deep_research_report] thread=%s ignoring retrieve=true; writing-mode pipeline writes from conversation materials only",
|
||||
thread_id,
|
||||
)
|
||||
materials_provider: Any = StaticHarvestMaterialProvider(materials)
|
||||
@@ -520,11 +548,7 @@ async def deep_research_report_tool(
|
||||
)
|
||||
|
||||
source_count = len(result.source_ids or [])
|
||||
summary = (
|
||||
f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,"
|
||||
f"已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。"
|
||||
"请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
|
||||
)
|
||||
summary = f"研究报告已完成:基于 {source_count} 条资料({material_source})按「通用分析结构」撰写,已写入 {REPORT_VIRTUAL_PATH} 并展示给用户。请用 2-3 句话向用户简要总结报告的核心结论,不要复述全文。"
|
||||
logger.info(
|
||||
"[deep_research_report] thread=%s completed (sources=%d, report_chars=%d)",
|
||||
thread_id,
|
||||
@@ -540,10 +564,10 @@ async def deep_research_report_tool(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HarvestPlusSearchMaterialProvider",
|
||||
"REPORT_VIRTUAL_PATH",
|
||||
"StreamWriterEventSink",
|
||||
"build_writing_research_config",
|
||||
"deep_research_report_tool",
|
||||
"is_writing_mode_report_path",
|
||||
"topic_anchor_index",
|
||||
]
|
||||
Reference in new issue
Block a user