sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -13,6 +13,7 @@ don't carry those ids around.
from __future__ import annotations
import asyncio
import json
import logging
from collections.abc import Awaitable, Callable
@@ -20,7 +21,7 @@ from contextlib import suppress
from datetime import date, datetime
from enum import Enum
from time import monotonic
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, TypeVar
from deerflow.agents.deep_research.events import (
DeepResearchEvent,
@@ -33,6 +34,33 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_EVENT_PERSIST_TIMEOUT_SECONDS = 2.0
_T = TypeVar("_T")
async def _await_or_abandon(awaitable: Awaitable[_T], timeout: float) -> _T:
"""Return the result, or raise TimeoutError without waiting out a hung DB call.
``asyncio.wait_for`` cancels the inner task and then *awaits that cancel*.
On the production MySQL this second wait is what freezes writing after the
first live frame: the query never actually cancels, so the writer never
resumes. Abandon the task instead and continue live-only.
"""
task = asyncio.ensure_future(awaitable)
done, _ = await asyncio.wait({task}, timeout=timeout)
if task in done:
return task.result()
def _log_abandoned(done_task: asyncio.Task[_T]) -> None:
if done_task.cancelled():
return
exc = done_task.exception()
if exc is not None:
logger.warning("abandoned deep-research awaitable failed later: %s", exc)
task.add_done_callback(_log_abandoned)
raise TimeoutError(f"operation exceeded {timeout:g} seconds")
def _json_default(value: Any) -> Any:
if isinstance(value, (datetime, date)):
@@ -115,12 +143,15 @@ class PersistingEventSink:
# instead of leaving it buffered until the job ends.
await self._flush_live_mirrors()
try:
row = await self._repo.append(
session_id=self._session_id,
job_id=self._job_id,
event_type=type,
phase=phase,
payload=event_payload,
row = await _await_or_abandon(
self._repo.append(
session_id=self._session_id,
job_id=self._job_id,
event_type=type,
phase=phase,
payload=event_payload,
),
_EVENT_PERSIST_TIMEOUT_SECONDS,
)
# Reconstruct the envelope from the persisted row (seq is now assigned).
event = DeepResearchEvent(
@@ -238,8 +269,8 @@ class PersistingEventSink:
type,
exc_info=True,
)
with suppress(Exception):
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
# Do not persist live frames. Mid-write event-table IO is what froze
# the production writer; the completed report is stored afterwards.
# ── cross-worker mirror for live-only reasoning frames ──────────────────
@@ -330,8 +330,13 @@ class DeerFlowCompletionBackend:
durable runner either uses its safe fallback or records a terminal error.
"""
timeout = self._config.control_plane_timeout_seconds
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
try:
return await asyncio.wait_for(model.ainvoke(messages, **kwargs), timeout=timeout)
return await asyncio.wait_for(
model.ainvoke(messages, config=isolated_config, **kwargs),
timeout=timeout,
)
except TimeoutError as exc:
if self._cancel is not None:
self._cancel.raise_if_cancelled()
@@ -340,6 +345,8 @@ class DeerFlowCompletionBackend:
code="completion_timeout",
retryable=True,
) from exc
finally:
reset_parent_config()
async def stream_complete(
self,
@@ -129,6 +129,7 @@ def harvest_messages_materials(
messages: list[Any],
*,
query: str,
include_rag_sources: bool = False,
) -> tuple[list[ResearchMaterial], str]:
"""Convert an in-memory message list into research materials.
@@ -141,6 +142,16 @@ def harvest_messages_materials(
the thread produced no harvestable tool results.
"""
materials = _harvest_tool_messages(messages, query=query)
if include_rag_sources:
# Gateway project retrieval is attached to the human turn, rather than
# emitted as a ToolMessage. Carry those verified sources into writing.
for message_index, message in enumerate(messages):
extra = message.get("additional_kwargs", {}) if isinstance(message, dict) else getattr(message, "additional_kwargs", {})
sources = (extra or {}).get("llmwiki_reference_sources")
if isinstance(sources, list):
materials.extend(_harvest_generic_payload(
{"results": sources}, tool_name="llmwiki_search", query=query, message_index=message_index,
))
agent_summary = _last_assistant_text(messages)
if not materials and agent_summary.strip():
# Weak-model fallback: the agent narrated its findings but never called
@@ -29,6 +29,10 @@ SourceType = Literal["web", "knowledge", "skill", "mcp", "file", "other"]
# and the writing job harvests materials from that thread; ``legacy`` keeps the
# runner-driven skill/web tiers searching on their own.
CollectionMode = Literal["chat", "legacy"]
# Who drafts the report skeleton when the user did not pick a preset structure.
# ``agent`` — dedicated planner agent runs before collection.
# ``pipeline`` — the writing job generates the outline itself (original fallback).
OutlinePlanner = Literal["agent", "pipeline"]
# How the writer treats ``custom_outline``.
# ``fixed`` (固定大纲) reproduces the stored headings and hierarchy.
@@ -65,6 +69,8 @@ class DeepResearchConfig(BaseModel):
mode: ResearchMode = "basic"
collection_mode: CollectionMode = "chat"
# Missing snapshots (older sessions) stay on the writer fallback.
outline_planner: OutlinePlanner = "pipeline"
language: str = "zh-CN"
tone: ResearchTone = "objective"
report_format: ReportFormat = "markdown"
@@ -184,6 +190,22 @@ class DeepResearchConfig(BaseModel):
cleaned = str(value).replace("\x00", "").replace("\r", "").strip()
return cleaned or None
@field_validator("outline_planner", mode="before")
@classmethod
def _normalize_outline_planner(cls, value: Any) -> OutlinePlanner:
raw = str(value or "").strip().lower()
if raw in {
"agent",
"planner",
"智能体",
"智能体规划",
"先行规划大纲",
"提纲预编制",
"研究提纲预编制",
}:
return "agent"
return "pipeline"
@field_validator("structure_mode", mode="before")
@classmethod
def _normalize_structure_mode(cls, value: Any) -> StructureMode:
@@ -225,6 +247,13 @@ ALL_MODES: tuple[ResearchMode, ...] = (
)
def config_from_dict(data: dict[str, Any]) -> DeepResearchConfig:
"""Build + clamp a config from a raw dict (e.g. JSON column / request body)."""
return DeepResearchConfig.model_validate(data).clamp()
def config_from_dict(data: dict[str, Any] | None) -> DeepResearchConfig:
"""Build + clamp a config from a raw dict (e.g. JSON column / request body).
Persistence metadata (``report_variant``, ``parent_session_id``) and any
other unknown keys are dropped before validation so a newer client cannot
500 an older worker — ``extra="forbid"`` stays for direct model construction.
"""
raw = data if isinstance(data, dict) else {}
payload = {key: value for key, value in raw.items() if key in DeepResearchConfig.model_fields}
return DeepResearchConfig.model_validate(payload).clamp()
@@ -55,6 +55,7 @@ __all__ = [
"normalize_retrieval_directions",
"normalize_retrieval_skills",
"queries_from_retrieval_directions",
"resolve_collector_skill_visibility",
"runtime_extra_skills",
]
@@ -120,10 +121,12 @@ def merge_collector_skill_allowlist(
base: list[str] | None,
extra: Any,
) -> list[str] | None:
"""Widen the collector agent's skill allowlist for one run.
"""Apply one run's 检索来源 extras to the collector agent's skill visibility.
Other agents are unchanged. Empty extras are a no-op. ``None`` base means
"all skills" (default lead agent) and is left alone.
The collector's base is ``None`` ("all skills", like the default lead
agent — its config.yaml whitelist is ignored). Empty extras keep ``None``;
non-empty extras narrow the run to exactly those skills, because 检索来源
is the user's explicit per-research choice. Other agents are unchanged.
"""
extras = normalize_retrieval_skills(extra)
if not extras or agent_id != COLLECTOR_AGENT_ID:
@@ -136,3 +139,20 @@ def merge_collector_skill_allowlist(
seen.add(name)
out.append(name)
return out
def resolve_collector_skill_visibility(
agent_id: str | None,
base: list[str] | None,
extra: Any = None,
) -> list[str] | None:
"""Resolve the collector's runtime skill visibility.
The deep-research collector does NOT use its config.yaml ``skills``
whitelist: without run extras it sees **every** enabled skill (``None``),
exactly like the default lead agent in a normal chat — free to use any
skill it wants. Non-collector agents pass through ``base`` untouched.
"""
if agent_id != COLLECTOR_AGENT_ID:
return base
return merge_collector_skill_allowlist(agent_id, None, extra)
@@ -51,6 +51,23 @@ logger = logging.getLogger(__name__)
# chunk size so the retry produces the same event volume as a streamed write.
_RETRY_REPLAY_CHUNK = 480
_WRITE_REPORT_TIMEOUT_SECONDS = 180.0
_INTENT_OUTLINE_TIMEOUT_SECONDS = 45.0
_FALLBACK_INTENT_OUTLINE = """# 研究报告
# 引言
# 一、问题与事实基础
## 1.1 范围与依据
## 1.2 已有证据
# 二、分析与判断
## 2.1 形成机制
## 2.2 影响与分歧
# 三、结论与后续关注
## 3.1 核心结论
## 3.2 资料局限
"""
async def _await_writer(coro):
@@ -105,6 +122,20 @@ _GEN_DIRECTED_QUERIES_PROMPT = """\
请以 JSON 格式返回,格式为:{{"queries": ["子问题1", "子问题2", ...]}}。
只返回 JSON,不要有其他文字。"""
_GEN_INTENT_OUTLINE_PROMPT = """\
研究课题:{query}
已收集资料要点(只用来判断该写哪些章节,不要编造资料中没有的事实):
{material_digest}
用户没有选用预设报告结构。请根据课题意图生成一份适合撰写研究报告的 Markdown 多级大纲。
要求:
1. 只用 Markdown 标题:`#` 题名、`##` 大章、`###` 小章;不要写正文、不要解释、不要代码围栏。
2. 大章 1–6 个,标题必须含本课题的专名或核心问题,不要套「概述 / 现状 / 问题 / 建议 / 结论」空壳。
3. 除引言、结论外,每个大章必须有 2–4 个 `###` 小章(1.1 / 1.2…),小章标题写成该节要回答的具体问题。
4. 禁止只有「一、二、三、四、五」的扁平列表。
5. 只返回大纲本身。"""
_REPORT_PROMPT = """\
请根据以下研究材料,撰写一份关于「{query}」的研究报告。
@@ -452,6 +483,94 @@ class BasicRunner:
cleaned = [q.strip() for q in queries if q.strip()][:n]
return cleaned or queries_from_retrieval_directions(query, dirs)[:n]
async def _generate_intent_outline(
self,
llm,
query: str,
context_text: str,
events=None,
cancellation=None,
) -> str:
"""Build a topic-specific outline when the user did not pick a structure.
Stream the planning tokens into the write trace so an empty confirmation
card does not look frozen. Bound the call so a hung strategic model
cannot stall the report itself; writing then continues on a fallback
skeleton.
"""
if cancellation:
cancellation.raise_if_cancelled()
if events is not None:
await events.emit(
"warning",
phase="writing",
payload={
"code": "intent_outline_started",
"message": "未选择报告结构,正在根据课题和材料规划章节…",
"recoverable": True,
},
)
async def on_delta(delta: str) -> None:
if events is not None:
await events.publish_live(
"report_thinking",
phase="writing",
payload={"delta": delta},
)
async def on_reasoning(delta: str) -> None:
if events is not None:
await events.publish_live(
"report_thinking",
phase="writing",
payload={"delta": delta},
)
prompt = build_intent_outline_prompt(query, context_text)
try:
result = await asyncio.wait_for(
llm.stream_complete(
model_role="strategic",
messages=[{"role": "system", "content": _SYSTEM}, {"role": "user", "content": prompt}],
operation="generate_intent_outline",
on_delta=on_delta,
on_reasoning=on_reasoning,
),
timeout=_INTENT_OUTLINE_TIMEOUT_SECONDS,
)
except ResearchCancelled:
raise
except Exception:
logger.warning(
"basic runner: intent outline generation failed; writing with a fallback skeleton",
exc_info=True,
)
return await self._fallback_intent_outline(events)
outline = extract_generated_outline(result.text)
if not outline:
return await self._fallback_intent_outline(events)
if events is not None:
await events.emit(
"plan_created",
phase="writing",
payload={"outline": outline, "source": "intent"},
)
return outline
async def _fallback_intent_outline(self, events) -> str:
if events is not None:
await events.emit(
"warning",
phase="writing",
payload={
"code": "intent_outline_fallback",
"message": "未能根据课题生成报告结构,将按通用报告结构继续撰写。",
"recoverable": True,
},
)
return _FALLBACK_INTENT_OUTLINE
async def _search_one(
self,
provider,
@@ -624,6 +743,19 @@ class BasicRunner:
never hide a second model generation inside the same user action.
"""
outline = request.config.custom_outline
if not (outline or "").strip():
generated = await self._generate_intent_outline(
llm,
request.query,
context_text,
events,
cancellation,
)
if generated:
outline = generated
request = request.model_copy(
update={"config": request.config.model_copy(update={"custom_outline": generated})}
)
filled_preamble = fill_report_preamble(
extract_outline_preamble(outline),
query=request.query,
@@ -872,6 +1004,66 @@ class BasicRunner:
# ── helpers ─────────────────────────────────────────────────────────────────
_OUTLINE_FENCE_RE = re.compile(r"```(?:markdown|md)?\s*([\s\S]*?)```", re.IGNORECASE)
_OUTLINE_HEADING_RE = re.compile(r"^#{1,6}\s+\S")
_OUTLINE_NUM_SUB_RE = re.compile(r"^\d+\.\d+(?:\.\d+)*[、..]?\s*\S")
_OUTLINE_PAREN_SUB_RE = re.compile(r"^[((][一二三四五六七八九十]+[))]\s*\S")
_OUTLINE_CN_MAJOR_RE = re.compile(r"^(?:第)?[一二三四五六七八九十]+[、..]\s*\S")
_OUTLINE_NUM_MAJOR_RE = re.compile(r"^\d+[、..]\s*\S")
def build_intent_outline_prompt(query: str, context_text: str) -> str:
"""User prompt for a topic-specific outline after collection."""
return _GEN_INTENT_OUTLINE_PROMPT.format(
query=query,
material_digest=_material_digest_for_outline(context_text),
)
def _material_digest_for_outline(context_text: str, *, limit: int = 1_800) -> str:
digest = (context_text or "").strip()
if not digest:
return "(暂无已收集资料摘要,请仅根据课题意图规划结构)"
compact = re.sub(r"\n{3,}", "\n\n", digest)
if len(compact) <= limit:
return compact
return compact[:limit].rstrip() + "…"
def _normalize_outline_heading_line(line: str) -> str | None:
"""Keep Markdown headings; lift numbered chapter lines into ## / ###."""
text = line.strip()
if not text:
return None
if _OUTLINE_HEADING_RE.match(text):
return text
if _OUTLINE_NUM_SUB_RE.match(text) or _OUTLINE_PAREN_SUB_RE.match(text):
return f"### {text}"
if _OUTLINE_CN_MAJOR_RE.match(text) or _OUTLINE_NUM_MAJOR_RE.match(text):
return f"## {text}"
return None
def extract_generated_outline(text: str | None) -> str | None:
"""Keep only a Markdown heading skeleton from a free-form model reply."""
raw = (text or "").strip()
if not raw:
return None
fenced = _OUTLINE_FENCE_RE.search(raw)
if fenced:
raw = fenced.group(1).strip()
headings = [
normalized
for line in raw.splitlines()
if (normalized := _normalize_outline_heading_line(line))
]
if len(headings) >= 2:
return "\n".join(headings)
if raw.startswith("#"):
return raw
return None
def _report_outline_instruction(custom_outline: str | None) -> str:
"""Render a user outline as the required skeleton, not as extra commands."""
if not custom_outline:
@@ -1,9 +1,8 @@
"""Small, durable report-delta emitter shared by research runners.
"""Live report-delta emitter shared by research runners.
The event store intentionally persists every emitted event. Sending one event
for each provider token would therefore create excessive database writes and
make reconnect replay expensive. This helper keeps the first visible delta
immediate, then coalesces later deltas into bounded chunks.
Writing must stay visible even when MySQL is slow. Provider tokens go to the
SSE hub immediately. The completed ``report_markdown`` is stored on the session
after the model finishes — never as mid-stream ``report_chunk`` rows.
"""
from __future__ import annotations
@@ -32,17 +31,16 @@ class ReportChunkEmitter:
self._phase = phase
self._chunk_size = chunk_size
self._buffer = ""
self._sent_first = False
self._next_offset = 0
self._persisted_offset = 0
async def add(self, delta: str) -> None:
"""Add a provider delta, making the first one visible immediately."""
"""Add a provider delta, making it visible immediately."""
if not delta:
return
# The sandbox UI receives the native model delta immediately. This is
# intentionally non-durable; ``report_chunk`` below remains the
# bounded checkpoint used for reconnect replay.
# Must not await event-store IO here. This callback runs inside
# ``model.astream``; a hung append pauses HTTP reads and the writing
# connection dies after the first paragraph on the production server.
await self._events.publish_live(
"report_delta",
phase=self._phase,
@@ -55,14 +53,6 @@ class ReportChunkEmitter:
)
self._next_offset += len(delta)
self._buffer += delta
if not self._sent_first:
first = self._buffer[: self._chunk_size]
await self._emit(first)
self._buffer = self._buffer[self._chunk_size :]
self._sent_first = True
while len(self._buffer) >= self._chunk_size:
await self._emit(self._buffer[: self._chunk_size])
self._buffer = self._buffer[self._chunk_size :]
async def add_thinking(self, delta: str) -> None:
"""Forward reasoning to the live UI without persisting it as report text."""
@@ -79,23 +69,8 @@ class ReportChunkEmitter:
)
async def flush(self) -> None:
"""Persist the final tail once the model stream has finished."""
if self._buffer:
await self._emit(self._buffer)
self._buffer = ""
async def _emit(self, delta: str) -> None:
await self._events.emit(
"report_chunk",
phase=self._phase,
payload={
"delta": delta,
"target": self._target,
"reportVersion": self._report_version,
"offset": self._persisted_offset,
},
)
self._persisted_offset += len(delta)
"""Drop the in-memory buffer. Persistence happens after the report exists."""
self._buffer = ""
__all__ = ["ReportChunkEmitter"]