初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1 @@
"""Chat collection package — harvest materials from a collector-agent thread."""
@@ -0,0 +1,311 @@
"""Harvest research materials from a collector-agent chat thread.
The chat collection flow replaces the runner-driven search tiers with a real
collector-agent conversation on the session's runtime LangGraph thread (same
pipeline as normal chat — skills, sandbox tools, repair middlewares all work
with weak models). When the user confirms report writing, the writing job
reads that thread's checkpoint and converts the tool results the agent already
produced into :class:`ResearchMaterial` objects:
- ``web_search`` tool envelopes reuse the exact configured-search field
mapping (``result_to_material``) so citations stay identical to the legacy
channel.
- other tool outputs (knowledge/skill/MCP/bash stdout, including
middle-truncated JSON) are normalized through ``deerflow.runtime.references``
— the same robust harvester the chat reference batches use.
- the agent's final collection summary becomes a fallback material so a weak
model that finished without usable tool output still yields a report.
The harvested pool is served to the runners through
:class:`StaticHarvestMaterialProvider` — runners and their dedupe/compress/
write stages stay untouched.
"""
from __future__ import annotations
import hashlib
import logging
from datetime import datetime
from typing import Any
from deerflow.agents.deep_research.adapters.material import dedupe_materials
from deerflow.agents.deep_research.adapters.material_provider import result_to_material
from deerflow.agents.deep_research.types import ResearchMaterial
from deerflow.runtime.references import (
_extract_tool_payload,
_first_list,
_message_content,
_message_name,
_message_type,
_normalize_reference_item,
)
logger = logging.getLogger(__name__)
# Upper bound on harvested materials so a chatty collector thread cannot blow
# up the report context; the curating/compressing stages trim further anyway.
MAX_HARVESTED_MATERIALS = 80
# A collector model may time out after tool calls were checkpointed but before
# its final assistant response advanced the thread head. Keep the scan bounded
# while retaining the richest checkpoint from that short history.
MAX_CHECKPOINT_HISTORY = 32
# Tool-name → ResearchMaterial.source_type. Everything unknown maps to
# ``other``; ``bash`` is treated as ``skill`` because skill scripts emit there.
_TOOL_SOURCE_TYPES: dict[str, str] = {
"web_search": "web",
"weknora_search": "knowledge",
"bash": "skill",
"read_file": "file",
"write_file": "file",
}
# Tool results that never carry report materials.
_IGNORED_TOOLS = {"ask_clarification", "write_todos", "deep_research_report"}
async def harvest_thread_materials(
*,
checkpointer: Any,
runtime_thread_id: str,
query: str,
) -> tuple[list[ResearchMaterial], str]:
"""Read the collector thread's checkpointed messages into materials.
Returns ``(materials, agent_summary)`` — the summary is the agent's last
visible text (its collection小结) and doubles as a fallback material when
the thread produced no harvestable tool results.
"""
config = {"configurable": {"thread_id": runtime_thread_id, "checkpoint_ns": ""}}
checkpoint_tuple = await checkpointer.aget_tuple(config)
if checkpoint_tuple is None:
logger.warning("deep-research harvest: no checkpoint for thread %s", runtime_thread_id)
return [], ""
best_materials, best_summary = _harvest_checkpoint_tuple(checkpoint_tuple, query=query)
# The latest checkpoint is the normal, fast path. Historical traversal is
# only a recovery for a stale/empty head after an interrupted collector
# stream. Scanning it after usable material was already found adds dozens
# of checkpoint reads and can make report writing appear to stop halfway
# on deployments whose checkpoint store is slow or isolated per worker.
if best_materials:
return best_materials, best_summary
# ``aget_tuple`` normally returns the latest state. When a streamed lead
# agent call fails, however, some checkpointers leave that head at the
# final input checkpoint even though an earlier checkpoint already holds
# the successful web-search tool results. With an empty head, prefer the
# richest available material pool rather than treating it as an empty run.
checkpoint_history = getattr(checkpointer, "alist", None)
if callable(checkpoint_history):
try:
history = checkpoint_history(config, limit=MAX_CHECKPOINT_HISTORY)
if hasattr(history, "__aiter__"):
async for candidate in history:
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
if len(materials) > len(best_materials):
best_materials, best_summary = materials, summary
else:
for candidate in history:
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
if len(materials) > len(best_materials):
best_materials, best_summary = materials, summary
except Exception: # noqa: BLE001 - newest checkpoint remains a safe fallback
logger.debug("deep-research harvest: checkpoint history unavailable for %s", runtime_thread_id, exc_info=True)
return best_materials, best_summary
def _harvest_checkpoint_tuple(checkpoint_tuple: Any, *, query: str) -> tuple[list[ResearchMaterial], str]:
checkpoint = getattr(checkpoint_tuple, "checkpoint", None)
if not isinstance(checkpoint, dict):
return [], ""
values = checkpoint.get("channel_values", {}) or {}
messages = values.get("messages") or []
return harvest_messages_materials(messages, query=query)
def harvest_messages_materials(
messages: list[Any],
*,
query: str,
) -> tuple[list[ResearchMaterial], str]:
"""Convert an in-memory message list into research materials.
Same conversion as :func:`harvest_thread_materials` minus the checkpoint
read — for callers that already hold the messages (e.g. a tool reading
``runtime.state`` inside the same run).
Returns ``(materials, agent_summary)`` — the summary is the agent's last
visible text (its collection小结) and doubles as a fallback material when
the thread produced no harvestable tool results.
"""
materials = _harvest_tool_messages(messages, query=query)
agent_summary = _last_assistant_text(messages)
if not materials and agent_summary.strip():
# Weak-model fallback: the agent narrated its findings but never called
# a harvestable tool — its summary is still real collected knowledge.
materials.append(
ResearchMaterial(
id=f"src_{hashlib.sha256(agent_summary.encode('utf-8')).hexdigest()[:16]}",
query=query,
title="收集员小结",
url=None,
raw_content=agent_summary,
snippet=agent_summary[:200],
source="collector",
source_type="other",
content_hash=hashlib.sha256(agent_summary.encode("utf-8")).hexdigest()[:32],
metadata={"retrieval_channel": "chat_collector_summary"},
)
)
return dedupe_materials(materials)[:MAX_HARVESTED_MATERIALS], agent_summary
def _harvest_tool_messages(messages: list[Any], *, query: str) -> list[ResearchMaterial]:
"""Convert every harvestable tool result in the thread into materials.
每条材料在 ``metadata["message_index"]`` 记录其来源工具消息在对话中的
位置——写作管线据此区分「课题请求之后检索的(属于当前课题)」与
「课题切换前的旧检索」(后者才做词面相关度过滤)。
"""
materials: list[ResearchMaterial] = []
for message_index, message in enumerate(messages):
if _message_type(message) != "tool":
continue
tool_name = str(_message_name(message) or "")
if tool_name in _IGNORED_TOOLS:
continue
payload = _extract_tool_payload(message)
if payload is None:
continue
if tool_name == "web_search" and isinstance(payload, dict):
# Highest-fidelity path: identical field mapping to the legacy
# configured-search channel (recUuid/url/content/publish_time).
envelope_query = str(payload.get("query") or "").strip() or query
for item in payload.get("results") or []:
if isinstance(item, dict):
material = result_to_material(item, query=envelope_query)
material.metadata["retrieval_channel"] = "chat_web_search"
material.metadata["message_index"] = message_index
materials.append(material)
continue
materials.extend(_harvest_generic_payload(payload, tool_name=tool_name, query=query, message_index=message_index))
return materials
def _harvest_generic_payload(
payload: Any,
*,
tool_name: str,
query: str,
message_index: int | None = None,
) -> list[ResearchMaterial]:
"""Normalize a non-web_search tool payload (skill/knowledge/bash/MCP)."""
source_type = _TOOL_SOURCE_TYPES.get(tool_name, "other")
materials: list[ResearchMaterial] = []
for index, raw_item in enumerate(_first_list(payload), 1):
normalized = _normalize_reference_item(raw_item, index, payload=payload, tool_context=None)
if not normalized:
continue
title = str(normalized.get("title") or "").strip()
content = str(normalized.get("content") or normalized.get("snippet") or "").strip()
url = str(normalized.get("url") or "").strip() or None
if not title and not content:
continue
raw_hash = hashlib.sha256((url or title or content).encode("utf-8")).hexdigest()[:32]
materials.append(
ResearchMaterial(
id=f"src_{raw_hash[:16]}",
query=query,
title=title or f"结果 {index}",
url=url,
raw_content=content,
snippet=content[:200] or None,
source=str(normalized.get("source") or tool_name).strip() or tool_name,
source_type=source_type, # type: ignore[arg-type]
published_at=_try_parse_datetime(str(normalized.get("time") or "")),
content_hash=raw_hash,
metadata={
"retrieval_channel": "chat_tool",
"tool_name": tool_name,
**({"message_index": message_index} if message_index is not None else {}),
},
)
)
return materials
def _last_assistant_text(messages: list[Any]) -> str:
"""The collector's last visible answer (its collection summary)."""
for message in reversed(messages):
if _message_type(message) != "ai":
continue
content = _message_content(message)
text = _content_to_text(content)
if text.strip():
return text.strip()
return ""
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for part in content:
if isinstance(part, str):
parts.append(part)
elif isinstance(part, dict) and isinstance(part.get("text"), str):
parts.append(part["text"])
return "".join(parts)
return ""
def _try_parse_datetime(value: str) -> datetime | None:
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S"):
try:
return datetime.strptime(value[:19], fmt)
except ValueError:
continue
return None
class StaticHarvestMaterialProvider:
"""MaterialProvider serving the pre-harvested collector-thread pool.
Runners call ``search`` per planned sub-query; for a chat-collected pool
every call returns the whole harvest (the runners' dedupe + compress
stages trim it). ``limit`` therefore does not truncate — it bounds
network fan-out, which this provider never performs.
"""
def __init__(self, materials: list[ResearchMaterial]) -> None:
self._materials = list(materials)
@property
def material_count(self) -> int:
return len(self._materials)
async def search(
self,
query: str,
*,
domains: list[str] | None = None,
limit: int = 8,
user_id: str = "",
session_id: str = "",
cancellation=None,
) -> list[ResearchMaterial]:
if cancellation is not None:
cancellation.raise_if_cancelled()
return list(self._materials)
__all__ = [
"MAX_CHECKPOINT_HISTORY",
"MAX_HARVESTED_MATERIALS",
"StaticHarvestMaterialProvider",
"harvest_messages_materials",
"harvest_thread_materials",
]