sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+40
-9
@@ -13,6 +13,7 @@ don't carry those ids around.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
@@ -20,7 +21,7 @@ from contextlib import suppress
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
|
||||
from deerflow.agents.deep_research.events import (
|
||||
DeepResearchEvent,
|
||||
@@ -33,6 +34,33 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EVENT_PERSIST_TIMEOUT_SECONDS = 2.0
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
async def _await_or_abandon(awaitable: Awaitable[_T], timeout: float) -> _T:
|
||||
"""Return the result, or raise TimeoutError without waiting out a hung DB call.
|
||||
|
||||
``asyncio.wait_for`` cancels the inner task and then *awaits that cancel*.
|
||||
On the production MySQL this second wait is what freezes writing after the
|
||||
first live frame: the query never actually cancels, so the writer never
|
||||
resumes. Abandon the task instead and continue live-only.
|
||||
"""
|
||||
task = asyncio.ensure_future(awaitable)
|
||||
done, _ = await asyncio.wait({task}, timeout=timeout)
|
||||
if task in done:
|
||||
return task.result()
|
||||
|
||||
def _log_abandoned(done_task: asyncio.Task[_T]) -> None:
|
||||
if done_task.cancelled():
|
||||
return
|
||||
exc = done_task.exception()
|
||||
if exc is not None:
|
||||
logger.warning("abandoned deep-research awaitable failed later: %s", exc)
|
||||
|
||||
task.add_done_callback(_log_abandoned)
|
||||
raise TimeoutError(f"operation exceeded {timeout:g} seconds")
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
if isinstance(value, (datetime, date)):
|
||||
@@ -115,12 +143,15 @@ class PersistingEventSink:
|
||||
# instead of leaving it buffered until the job ends.
|
||||
await self._flush_live_mirrors()
|
||||
try:
|
||||
row = await self._repo.append(
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
event_type=type,
|
||||
phase=phase,
|
||||
payload=event_payload,
|
||||
row = await _await_or_abandon(
|
||||
self._repo.append(
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
event_type=type,
|
||||
phase=phase,
|
||||
payload=event_payload,
|
||||
),
|
||||
_EVENT_PERSIST_TIMEOUT_SECONDS,
|
||||
)
|
||||
# Reconstruct the envelope from the persisted row (seq is now assigned).
|
||||
event = DeepResearchEvent(
|
||||
@@ -238,8 +269,8 @@ class PersistingEventSink:
|
||||
type,
|
||||
exc_info=True,
|
||||
)
|
||||
with suppress(Exception):
|
||||
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
|
||||
# Do not persist live frames. Mid-write event-table IO is what froze
|
||||
# the production writer; the completed report is stored afterwards.
|
||||
|
||||
# ── cross-worker mirror for live-only reasoning frames ──────────────────
|
||||
|
||||
|
||||
+8
-1
@@ -330,8 +330,13 @@ class DeerFlowCompletionBackend:
|
||||
durable runner either uses its safe fallback or records a terminal error.
|
||||
"""
|
||||
timeout = self._config.control_plane_timeout_seconds
|
||||
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
|
||||
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
|
||||
try:
|
||||
return await asyncio.wait_for(model.ainvoke(messages, **kwargs), timeout=timeout)
|
||||
return await asyncio.wait_for(
|
||||
model.ainvoke(messages, config=isolated_config, **kwargs),
|
||||
timeout=timeout,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
@@ -340,6 +345,8 @@ class DeerFlowCompletionBackend:
|
||||
code="completion_timeout",
|
||||
retryable=True,
|
||||
) from exc
|
||||
finally:
|
||||
reset_parent_config()
|
||||
|
||||
async def stream_complete(
|
||||
self,
|
||||
|
||||
+11
@@ -129,6 +129,7 @@ def harvest_messages_materials(
|
||||
messages: list[Any],
|
||||
*,
|
||||
query: str,
|
||||
include_rag_sources: bool = False,
|
||||
) -> tuple[list[ResearchMaterial], str]:
|
||||
"""Convert an in-memory message list into research materials.
|
||||
|
||||
@@ -141,6 +142,16 @@ def harvest_messages_materials(
|
||||
the thread produced no harvestable tool results.
|
||||
"""
|
||||
materials = _harvest_tool_messages(messages, query=query)
|
||||
if include_rag_sources:
|
||||
# Gateway project retrieval is attached to the human turn, rather than
|
||||
# emitted as a ToolMessage. Carry those verified sources into writing.
|
||||
for message_index, message in enumerate(messages):
|
||||
extra = message.get("additional_kwargs", {}) if isinstance(message, dict) else getattr(message, "additional_kwargs", {})
|
||||
sources = (extra or {}).get("llmwiki_reference_sources")
|
||||
if isinstance(sources, list):
|
||||
materials.extend(_harvest_generic_payload(
|
||||
{"results": sources}, tool_name="llmwiki_search", query=query, message_index=message_index,
|
||||
))
|
||||
agent_summary = _last_assistant_text(messages)
|
||||
if not materials and agent_summary.strip():
|
||||
# Weak-model fallback: the agent narrated its findings but never called
|
||||
|
||||
+32
-3
@@ -29,6 +29,10 @@ SourceType = Literal["web", "knowledge", "skill", "mcp", "file", "other"]
|
||||
# and the writing job harvests materials from that thread; ``legacy`` keeps the
|
||||
# runner-driven skill/web tiers searching on their own.
|
||||
CollectionMode = Literal["chat", "legacy"]
|
||||
# Who drafts the report skeleton when the user did not pick a preset structure.
|
||||
# ``agent`` — dedicated planner agent runs before collection.
|
||||
# ``pipeline`` — the writing job generates the outline itself (original fallback).
|
||||
OutlinePlanner = Literal["agent", "pipeline"]
|
||||
|
||||
# How the writer treats ``custom_outline``.
|
||||
# ``fixed`` (固定大纲) reproduces the stored headings and hierarchy.
|
||||
@@ -65,6 +69,8 @@ class DeepResearchConfig(BaseModel):
|
||||
|
||||
mode: ResearchMode = "basic"
|
||||
collection_mode: CollectionMode = "chat"
|
||||
# Missing snapshots (older sessions) stay on the writer fallback.
|
||||
outline_planner: OutlinePlanner = "pipeline"
|
||||
language: str = "zh-CN"
|
||||
tone: ResearchTone = "objective"
|
||||
report_format: ReportFormat = "markdown"
|
||||
@@ -184,6 +190,22 @@ class DeepResearchConfig(BaseModel):
|
||||
cleaned = str(value).replace("\x00", "").replace("\r", "").strip()
|
||||
return cleaned or None
|
||||
|
||||
@field_validator("outline_planner", mode="before")
|
||||
@classmethod
|
||||
def _normalize_outline_planner(cls, value: Any) -> OutlinePlanner:
|
||||
raw = str(value or "").strip().lower()
|
||||
if raw in {
|
||||
"agent",
|
||||
"planner",
|
||||
"智能体",
|
||||
"智能体规划",
|
||||
"先行规划大纲",
|
||||
"提纲预编制",
|
||||
"研究提纲预编制",
|
||||
}:
|
||||
return "agent"
|
||||
return "pipeline"
|
||||
|
||||
@field_validator("structure_mode", mode="before")
|
||||
@classmethod
|
||||
def _normalize_structure_mode(cls, value: Any) -> StructureMode:
|
||||
@@ -225,6 +247,13 @@ ALL_MODES: tuple[ResearchMode, ...] = (
|
||||
)
|
||||
|
||||
|
||||
def config_from_dict(data: dict[str, Any]) -> DeepResearchConfig:
|
||||
"""Build + clamp a config from a raw dict (e.g. JSON column / request body)."""
|
||||
return DeepResearchConfig.model_validate(data).clamp()
|
||||
def config_from_dict(data: dict[str, Any] | None) -> DeepResearchConfig:
|
||||
"""Build + clamp a config from a raw dict (e.g. JSON column / request body).
|
||||
|
||||
Persistence metadata (``report_variant``, ``parent_session_id``) and any
|
||||
other unknown keys are dropped before validation so a newer client cannot
|
||||
500 an older worker — ``extra="forbid"`` stays for direct model construction.
|
||||
"""
|
||||
raw = data if isinstance(data, dict) else {}
|
||||
payload = {key: value for key, value in raw.items() if key in DeepResearchConfig.model_fields}
|
||||
return DeepResearchConfig.model_validate(payload).clamp()
|
||||
+23
-3
@@ -55,6 +55,7 @@ __all__ = [
|
||||
"normalize_retrieval_directions",
|
||||
"normalize_retrieval_skills",
|
||||
"queries_from_retrieval_directions",
|
||||
"resolve_collector_skill_visibility",
|
||||
"runtime_extra_skills",
|
||||
]
|
||||
|
||||
@@ -120,10 +121,12 @@ def merge_collector_skill_allowlist(
|
||||
base: list[str] | None,
|
||||
extra: Any,
|
||||
) -> list[str] | None:
|
||||
"""Widen the collector agent's skill allowlist for one run.
|
||||
"""Apply one run's 检索来源 extras to the collector agent's skill visibility.
|
||||
|
||||
Other agents are unchanged. Empty extras are a no-op. ``None`` base means
|
||||
"all skills" (default lead agent) and is left alone.
|
||||
The collector's base is ``None`` ("all skills", like the default lead
|
||||
agent — its config.yaml whitelist is ignored). Empty extras keep ``None``;
|
||||
non-empty extras narrow the run to exactly those skills, because 检索来源
|
||||
is the user's explicit per-research choice. Other agents are unchanged.
|
||||
"""
|
||||
extras = normalize_retrieval_skills(extra)
|
||||
if not extras or agent_id != COLLECTOR_AGENT_ID:
|
||||
@@ -136,3 +139,20 @@ def merge_collector_skill_allowlist(
|
||||
seen.add(name)
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def resolve_collector_skill_visibility(
|
||||
agent_id: str | None,
|
||||
base: list[str] | None,
|
||||
extra: Any = None,
|
||||
) -> list[str] | None:
|
||||
"""Resolve the collector's runtime skill visibility.
|
||||
|
||||
The deep-research collector does NOT use its config.yaml ``skills``
|
||||
whitelist: without run extras it sees **every** enabled skill (``None``),
|
||||
exactly like the default lead agent in a normal chat — free to use any
|
||||
skill it wants. Non-collector agents pass through ``base`` untouched.
|
||||
"""
|
||||
if agent_id != COLLECTOR_AGENT_ID:
|
||||
return base
|
||||
return merge_collector_skill_allowlist(agent_id, None, extra)
|
||||
+587
-7
@@ -51,6 +51,23 @@ logger = logging.getLogger(__name__)
|
||||
# chunk size so the retry produces the same event volume as a streamed write.
|
||||
_RETRY_REPLAY_CHUNK = 480
|
||||
_WRITE_REPORT_TIMEOUT_SECONDS = 180.0
|
||||
_INTENT_OUTLINE_TIMEOUT_SECONDS = 45.0
|
||||
_FALLBACK_INTENT_OUTLINE = """# 研究报告
|
||||
|
||||
# 引言
|
||||
|
||||
# 一、问题与事实基础
|
||||
## 1.1 范围与依据
|
||||
## 1.2 已有证据
|
||||
|
||||
# 二、分析与判断
|
||||
## 2.1 形成机制
|
||||
## 2.2 影响与分歧
|
||||
|
||||
# 三、结论与后续关注
|
||||
## 3.1 核心结论
|
||||
## 3.2 资料局限
|
||||
"""
|
||||
|
||||
|
||||
async def _await_writer(coro):
|
||||
@@ -105,6 +122,20 @@ _GEN_DIRECTED_QUERIES_PROMPT = """\
|
||||
请以 JSON 格式返回,格式为:{{"queries": ["子问题1", "子问题2", ...]}}。
|
||||
只返回 JSON,不要有其他文字。"""
|
||||
|
||||
_GEN_INTENT_OUTLINE_PROMPT = """\
|
||||
研究课题:{query}
|
||||
|
||||
已收集资料要点(只用来判断该写哪些章节,不要编造资料中没有的事实):
|
||||
{material_digest}
|
||||
|
||||
用户没有选用预设报告结构。请根据课题意图生成一份适合撰写研究报告的 Markdown 多级大纲。
|
||||
要求:
|
||||
1. 只用 Markdown 标题:`#` 题名、`##` 大章、`###` 小章;不要写正文、不要解释、不要代码围栏。
|
||||
2. 大章 1–6 个,标题必须含本课题的专名或核心问题,不要套「概述 / 现状 / 问题 / 建议 / 结论」空壳。
|
||||
3. 除引言、结论外,每个大章必须有 2–4 个 `###` 小章(1.1 / 1.2…),小章标题写成该节要回答的具体问题。
|
||||
4. 禁止只有「一、二、三、四、五」的扁平列表。
|
||||
5. 只返回大纲本身。"""
|
||||
|
||||
_REPORT_PROMPT = """\
|
||||
请根据以下研究材料,撰写一份关于「{query}」的研究报告。
|
||||
|
||||
@@ -452,6 +483,94 @@ class BasicRunner:
|
||||
cleaned = [q.strip() for q in queries if q.strip()][:n]
|
||||
return cleaned or queries_from_retrieval_directions(query, dirs)[:n]
|
||||
|
||||
async def _generate_intent_outline(
|
||||
self,
|
||||
llm,
|
||||
query: str,
|
||||
context_text: str,
|
||||
events=None,
|
||||
cancellation=None,
|
||||
) -> str:
|
||||
"""Build a topic-specific outline when the user did not pick a structure.
|
||||
|
||||
Stream the planning tokens into the write trace so an empty confirmation
|
||||
card does not look frozen. Bound the call so a hung strategic model
|
||||
cannot stall the report itself; writing then continues on a fallback
|
||||
skeleton.
|
||||
"""
|
||||
if cancellation:
|
||||
cancellation.raise_if_cancelled()
|
||||
if events is not None:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="writing",
|
||||
payload={
|
||||
"code": "intent_outline_started",
|
||||
"message": "未选择报告结构,正在根据课题和材料规划章节…",
|
||||
"recoverable": True,
|
||||
},
|
||||
)
|
||||
|
||||
async def on_delta(delta: str) -> None:
|
||||
if events is not None:
|
||||
await events.publish_live(
|
||||
"report_thinking",
|
||||
phase="writing",
|
||||
payload={"delta": delta},
|
||||
)
|
||||
|
||||
async def on_reasoning(delta: str) -> None:
|
||||
if events is not None:
|
||||
await events.publish_live(
|
||||
"report_thinking",
|
||||
phase="writing",
|
||||
payload={"delta": delta},
|
||||
)
|
||||
|
||||
prompt = build_intent_outline_prompt(query, context_text)
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
llm.stream_complete(
|
||||
model_role="strategic",
|
||||
messages=[{"role": "system", "content": _SYSTEM}, {"role": "user", "content": prompt}],
|
||||
operation="generate_intent_outline",
|
||||
on_delta=on_delta,
|
||||
on_reasoning=on_reasoning,
|
||||
),
|
||||
timeout=_INTENT_OUTLINE_TIMEOUT_SECONDS,
|
||||
)
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"basic runner: intent outline generation failed; writing with a fallback skeleton",
|
||||
exc_info=True,
|
||||
)
|
||||
return await self._fallback_intent_outline(events)
|
||||
outline = extract_generated_outline(result.text)
|
||||
if not outline:
|
||||
return await self._fallback_intent_outline(events)
|
||||
if events is not None:
|
||||
await events.emit(
|
||||
"plan_created",
|
||||
phase="writing",
|
||||
payload={"outline": outline, "source": "intent"},
|
||||
)
|
||||
return outline
|
||||
|
||||
async def _fallback_intent_outline(self, events) -> str:
|
||||
if events is not None:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="writing",
|
||||
payload={
|
||||
"code": "intent_outline_fallback",
|
||||
"message": "未能根据课题生成报告结构,将按通用报告结构继续撰写。",
|
||||
"recoverable": True,
|
||||
},
|
||||
)
|
||||
return _FALLBACK_INTENT_OUTLINE
|
||||
|
||||
async def _search_one(
|
||||
self,
|
||||
provider,
|
||||
@@ -624,6 +743,19 @@ class BasicRunner:
|
||||
never hide a second model generation inside the same user action.
|
||||
"""
|
||||
outline = request.config.custom_outline
|
||||
if not (outline or "").strip():
|
||||
generated = await self._generate_intent_outline(
|
||||
llm,
|
||||
request.query,
|
||||
context_text,
|
||||
events,
|
||||
cancellation,
|
||||
)
|
||||
if generated:
|
||||
outline = generated
|
||||
request = request.model_copy(
|
||||
update={"config": request.config.model_copy(update={"custom_outline": generated})}
|
||||
)
|
||||
filled_preamble = fill_report_preamble(
|
||||
extract_outline_preamble(outline),
|
||||
query=request.query,
|
||||
@@ -692,6 +824,7 @@ class BasicRunner:
|
||||
message: str,
|
||||
error_type: str,
|
||||
error_detail: str,
|
||||
messages: list[dict[str, str]] | None = None,
|
||||
) -> str:
|
||||
"""Spend one real second model call before any deterministic report.
|
||||
|
||||
@@ -731,7 +864,7 @@ class BasicRunner:
|
||||
result = await _await_writer(
|
||||
complete_prose(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
messages=messages or completion_messages,
|
||||
operation="write_report_retry",
|
||||
)
|
||||
)
|
||||
@@ -744,7 +877,7 @@ class BasicRunner:
|
||||
result = await _await_writer(
|
||||
llm.stream_complete(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
messages=messages or completion_messages,
|
||||
operation="write_report_retry",
|
||||
on_delta=emitter.add,
|
||||
on_reasoning=emitter.add_thinking,
|
||||
@@ -795,7 +928,13 @@ class BasicRunner:
|
||||
payload={"reportVersion": report_version, "reason": reason},
|
||||
)
|
||||
fallback = ensure_outline_preamble(
|
||||
_build_report_fallback(request.query, context_text, source_count or 0),
|
||||
_build_report_fallback(
|
||||
request.query,
|
||||
context_text,
|
||||
source_count or 0,
|
||||
custom_outline=outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
),
|
||||
outline,
|
||||
request.query,
|
||||
)
|
||||
@@ -840,10 +979,76 @@ class BasicRunner:
|
||||
),
|
||||
report_version=3,
|
||||
)
|
||||
return report if not _is_refusal_report(report) else await refusal_fallback()
|
||||
if _is_refusal_report(report):
|
||||
return await refusal_fallback()
|
||||
outline_error = _outline_adherence_error(
|
||||
report,
|
||||
outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
)
|
||||
if outline_error:
|
||||
return await complete_with_fallback(
|
||||
reason="outline_mismatch_after_retry",
|
||||
message="报告写入重试完成但仍未遵循所选大纲,已按所选大纲生成保底稿",
|
||||
error_type="InvalidReportStructure",
|
||||
error_detail=outline_error,
|
||||
report_version=3,
|
||||
)
|
||||
return report
|
||||
|
||||
if not _is_refusal_report(report):
|
||||
return report
|
||||
outline_error = _outline_adherence_error(
|
||||
report,
|
||||
outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
)
|
||||
if not outline_error:
|
||||
return report
|
||||
|
||||
# A selected outline is a delivery contract, not merely a prompt
|
||||
# hint. Fixed mode validates the complete hierarchy; adaptive mode
|
||||
# only rejects zero core-heading overlap. Reuse the existing single
|
||||
# retry budget for one targeted rewrite; never add a third call.
|
||||
try:
|
||||
report = await retry_report(
|
||||
report_version=2,
|
||||
reason="outline_mismatch",
|
||||
code="report_outline_retry",
|
||||
message="生成结果未遵循所选报告大纲,正在按模板结构重新生成",
|
||||
error_type="InvalidReportStructure",
|
||||
error_detail=outline_error,
|
||||
messages=_outline_repair_messages(
|
||||
completion_messages,
|
||||
outline=outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
validation_error=outline_error,
|
||||
),
|
||||
)
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
except Exception as retry_exc: # noqa: BLE001 - retain a template-shaped result
|
||||
return await complete_with_fallback(
|
||||
reason="outline_mismatch_retry_failed",
|
||||
message="报告大纲重写未成功,已按所选大纲生成保底稿",
|
||||
error_type=type(retry_exc).__name__,
|
||||
error_detail=f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}",
|
||||
report_version=3,
|
||||
)
|
||||
|
||||
retry_outline_error = _outline_adherence_error(
|
||||
report,
|
||||
outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
)
|
||||
if not _is_refusal_report(report) and not retry_outline_error:
|
||||
return report
|
||||
return await complete_with_fallback(
|
||||
reason="outline_mismatch_after_retry",
|
||||
message="重写结果仍未遵循所选大纲,已按所选大纲生成保底稿",
|
||||
error_type="InvalidReportStructure",
|
||||
error_detail=retry_outline_error or report,
|
||||
report_version=3,
|
||||
)
|
||||
|
||||
# A refusal is a content failure, not a transport failure: one clean
|
||||
# rewrite recovers it far more often than a material dump does.
|
||||
@@ -866,12 +1071,285 @@ class BasicRunner:
|
||||
error_detail=f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}",
|
||||
report_version=3,
|
||||
)
|
||||
return report if not _is_refusal_report(report) else await refusal_fallback()
|
||||
if _is_refusal_report(report):
|
||||
return await refusal_fallback()
|
||||
outline_error = _outline_adherence_error(
|
||||
report,
|
||||
outline,
|
||||
structure_mode=request.config.structure_mode,
|
||||
)
|
||||
if outline_error:
|
||||
return await complete_with_fallback(
|
||||
reason="outline_mismatch_after_retry",
|
||||
message="报告重试结果未遵循所选大纲,已按所选大纲生成保底稿",
|
||||
error_type="InvalidReportStructure",
|
||||
error_detail=outline_error,
|
||||
report_version=3,
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
_OUTLINE_FENCE_RE = re.compile(r"```(?:markdown|md)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
_OUTLINE_HEADING_RE = re.compile(r"^#{1,6}\s+\S")
|
||||
_OUTLINE_NUM_SUB_RE = re.compile(r"^\d+\.\d+(?:\.\d+)*[、..]?\s*\S")
|
||||
_OUTLINE_PAREN_SUB_RE = re.compile(r"^[((][一二三四五六七八九十]+[))]\s*\S")
|
||||
_OUTLINE_CN_MAJOR_RE = re.compile(r"^(?:第)?[一二三四五六七八九十]+[、..]\s*\S")
|
||||
_OUTLINE_NUM_MAJOR_RE = re.compile(r"^\d+[、..]\s*\S")
|
||||
|
||||
|
||||
def build_intent_outline_prompt(query: str, context_text: str) -> str:
|
||||
"""User prompt for a topic-specific outline after collection."""
|
||||
return _GEN_INTENT_OUTLINE_PROMPT.format(
|
||||
query=query,
|
||||
material_digest=_material_digest_for_outline(context_text),
|
||||
)
|
||||
|
||||
|
||||
def _material_digest_for_outline(context_text: str, *, limit: int = 1_800) -> str:
|
||||
digest = (context_text or "").strip()
|
||||
if not digest:
|
||||
return "(暂无已收集资料摘要,请仅根据课题意图规划结构)"
|
||||
compact = re.sub(r"\n{3,}", "\n\n", digest)
|
||||
if len(compact) <= limit:
|
||||
return compact
|
||||
return compact[:limit].rstrip() + "…"
|
||||
|
||||
|
||||
def _normalize_outline_heading_line(line: str) -> str | None:
|
||||
"""Keep Markdown headings; lift numbered chapter lines into ## / ###."""
|
||||
text = line.strip()
|
||||
if not text:
|
||||
return None
|
||||
if _OUTLINE_HEADING_RE.match(text):
|
||||
return text
|
||||
if _OUTLINE_NUM_SUB_RE.match(text) or _OUTLINE_PAREN_SUB_RE.match(text):
|
||||
return f"### {text}"
|
||||
if _OUTLINE_CN_MAJOR_RE.match(text) or _OUTLINE_NUM_MAJOR_RE.match(text):
|
||||
return f"## {text}"
|
||||
return None
|
||||
|
||||
|
||||
def extract_generated_outline(text: str | None) -> str | None:
|
||||
"""Keep only a Markdown heading skeleton from a free-form model reply."""
|
||||
raw = (text or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
fenced = _OUTLINE_FENCE_RE.search(raw)
|
||||
if fenced:
|
||||
raw = fenced.group(1).strip()
|
||||
headings = [
|
||||
normalized
|
||||
for line in raw.splitlines()
|
||||
if (normalized := _normalize_outline_heading_line(line))
|
||||
]
|
||||
if len(headings) >= 2:
|
||||
return "\n".join(headings)
|
||||
if raw.startswith("#"):
|
||||
return raw
|
||||
return None
|
||||
|
||||
|
||||
_OUTLINE_TITLE_PLACEHOLDERS = frozenset(
|
||||
{
|
||||
"报告标题",
|
||||
"报告题目",
|
||||
"方案标题",
|
||||
"方案题目",
|
||||
"研究报告",
|
||||
}
|
||||
)
|
||||
|
||||
_ADAPTIVE_HEADING_PREFIX_RE = re.compile(
|
||||
r"^(?:"
|
||||
r"第\s*[零〇一二三四五六七八九十百千万\d]+\s*(?:章|节|部分|篇)\s*[、..::\-—]?\s*"
|
||||
r"|[((]\s*(?:[零〇一二三四五六七八九十百千万]+|\d+)\s*[))]\s*[、..::\-—]?\s*"
|
||||
r"|\d+(?:\.\d+)*(?:\s*[、..::\-—]\s*|\s+)"
|
||||
r"|[零〇一二三四五六七八九十百千万]+(?:\s*[、..::\-—]\s*|\s+)"
|
||||
r"|(?:chapter|part|section)\s+\d+(?:\s*[.::\-—]\s*|\s+)"
|
||||
r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_ADAPTIVE_GENERIC_HEADINGS = frozenset(
|
||||
{
|
||||
"摘要",
|
||||
"内容摘要",
|
||||
"执行摘要",
|
||||
"概述",
|
||||
"引言",
|
||||
"导言",
|
||||
"前言",
|
||||
"绪论",
|
||||
"结论",
|
||||
"核心结论",
|
||||
"主要结论",
|
||||
"研究结论",
|
||||
"结论与建议",
|
||||
"总结",
|
||||
"总结与展望",
|
||||
"结语",
|
||||
"参考资料",
|
||||
"参考来源",
|
||||
"参考文献",
|
||||
"附录",
|
||||
"附件",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalized_heading_title(value: str) -> str:
|
||||
"""Normalize harmless Markdown decoration without weakening title checks."""
|
||||
|
||||
text = re.sub(r"[`*_~]", "", value)
|
||||
return " ".join(text.split()).strip().casefold()
|
||||
|
||||
|
||||
def _adaptive_heading_key(value: str) -> str:
|
||||
"""Normalize chapter-number variants for loose adaptive matching only."""
|
||||
|
||||
text = _normalized_heading_title(value)
|
||||
while text:
|
||||
stripped = _ADAPTIVE_HEADING_PREFIX_RE.sub("", text, count=1).strip()
|
||||
if stripped == text:
|
||||
break
|
||||
text = stripped
|
||||
return text
|
||||
|
||||
|
||||
def _outline_heading_specs(value: str | None) -> list[tuple[int, str, str]]:
|
||||
"""Return ``(level, title, markdown_line)`` for a stored outline/report.
|
||||
|
||||
Report structures accept both Markdown headings and common Chinese/numbered
|
||||
chapter lines. Reusing ``_normalize_outline_heading_line`` keeps validation
|
||||
and the deterministic fallback aligned with the formats accepted by the
|
||||
writer prompt.
|
||||
"""
|
||||
|
||||
specs: list[tuple[int, str, str]] = []
|
||||
for raw_line in (value or "").splitlines():
|
||||
normalized = _normalize_outline_heading_line(raw_line)
|
||||
if not normalized:
|
||||
continue
|
||||
match = re.match(r"^(#{1,6})\s+(.+?)\s*$", normalized)
|
||||
if not match:
|
||||
continue
|
||||
title = match.group(2).strip()
|
||||
specs.append((len(match.group(1)), title, normalized))
|
||||
return specs
|
||||
|
||||
|
||||
def _outline_adherence_error(
|
||||
report: str,
|
||||
custom_outline: str | None,
|
||||
*,
|
||||
structure_mode: str,
|
||||
) -> str | None:
|
||||
"""Describe a selected-outline contract breach, or return ``None``.
|
||||
|
||||
Fixed outlines require every non-placeholder heading with its original
|
||||
level and order. Adaptive outlines permit material-driven changes, but a
|
||||
result with zero overlap against every core template chapter means the
|
||||
selected template was catastrophically ignored and warrants one rewrite.
|
||||
Extra generated headings remain allowed because the shared report contract
|
||||
may append a references/limitations section.
|
||||
"""
|
||||
|
||||
if not (custom_outline or "").strip():
|
||||
return None
|
||||
if structure_mode != "fixed":
|
||||
adaptive_specs = _fallback_outline_specs(
|
||||
custom_outline,
|
||||
structure_mode="adaptive",
|
||||
)
|
||||
if adaptive_specs:
|
||||
first, *remaining = adaptive_specs
|
||||
first_is_title = (
|
||||
_normalized_heading_title(first[1]) in _OUTLINE_TITLE_PLACEHOLDERS
|
||||
or (remaining and first[0] < min(level for level, _title, _line in remaining))
|
||||
)
|
||||
if first_is_title:
|
||||
adaptive_specs = remaining
|
||||
core = [
|
||||
(title, _adaptive_heading_key(title))
|
||||
for _level, title, _line in adaptive_specs
|
||||
if _adaptive_heading_key(title) not in _OUTLINE_TITLE_PLACEHOLDERS
|
||||
]
|
||||
core = [(title, key) for title, key in core if key]
|
||||
if not core:
|
||||
return None
|
||||
signal_core = [item for item in core if item[1] not in _ADAPTIVE_GENERIC_HEADINGS]
|
||||
expected = signal_core or core
|
||||
actual_titles = {
|
||||
_adaptive_heading_key(title)
|
||||
for _level, title, _line in _outline_heading_specs(report)
|
||||
}
|
||||
if any(key in actual_titles for _title, key in expected):
|
||||
return None
|
||||
shown = "、".join(title for title, _key in expected[:6])
|
||||
return f"弹性大纲未采用任何核心章节(应至少保留一项):{shown}"
|
||||
|
||||
required = [
|
||||
(level, title)
|
||||
for level, title, _line in _outline_heading_specs(custom_outline)
|
||||
if _normalized_heading_title(title) not in _OUTLINE_TITLE_PLACEHOLDERS
|
||||
]
|
||||
if not required:
|
||||
return None
|
||||
actual_normalized = [
|
||||
(level, _normalized_heading_title(title))
|
||||
for level, title, _line in _outline_heading_specs(report)
|
||||
]
|
||||
cursor = 0
|
||||
missing: list[str] = []
|
||||
for level, title in required:
|
||||
expected = (level, _normalized_heading_title(title))
|
||||
try:
|
||||
cursor = actual_normalized.index(expected, cursor) + 1
|
||||
except ValueError:
|
||||
missing.append(f"{'#' * level} {title}")
|
||||
if not missing:
|
||||
return None
|
||||
shown = "、".join(missing[:8])
|
||||
suffix = f"等 {len(missing)} 项" if len(missing) > 8 else ""
|
||||
return f"固定大纲缺少或打乱以下标题:{shown}{suffix}"
|
||||
|
||||
|
||||
def _outline_repair_messages(
|
||||
completion_messages: list[dict[str, str]],
|
||||
*,
|
||||
outline: str | None,
|
||||
structure_mode: str,
|
||||
validation_error: str,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Strengthen the sole retry after a selected-outline validation failure."""
|
||||
|
||||
messages = [dict(message) for message in completion_messages]
|
||||
if structure_mode == "fixed":
|
||||
repair = (
|
||||
"\n\n【固定大纲结构校验失败】\n"
|
||||
f"{validation_error}\n"
|
||||
"请从头完整重写报告。下面大纲中的每一个非占位标题都必须逐字保留,并严格按原顺序和层级出现;"
|
||||
"不得改名、合并、省略、降级、升级或调换。材料不足也要保留原标题,并在该节明确写明待补资料。\n"
|
||||
f"<required-fixed-outline>\n{outline or ''}\n</required-fixed-outline>"
|
||||
)
|
||||
else:
|
||||
repair = (
|
||||
"\n\n【弹性大纲结构校验失败】\n"
|
||||
f"{validation_error}\n"
|
||||
"请从头完整重写报告。可以按材料合并或省略没有依据的小节,但不能完全改用通用报告套式;"
|
||||
"至少保留下列大纲中的一个核心章节,并围绕所选模板组织其余内容。\n"
|
||||
f"<selected-adaptive-outline>\n{outline or ''}\n</selected-adaptive-outline>"
|
||||
)
|
||||
if messages:
|
||||
messages[-1]["content"] = str(messages[-1].get("content") or "") + repair
|
||||
else:
|
||||
messages.append({"role": "user", "content": repair.lstrip()})
|
||||
return messages
|
||||
|
||||
|
||||
def _report_outline_instruction(custom_outline: str | None) -> str:
|
||||
"""Render a user outline as the required skeleton, not as extra commands."""
|
||||
if not custom_outline:
|
||||
@@ -1214,7 +1692,98 @@ def _error_diagnostic(stage: str, exc: Exception, *, query: str | None = None) -
|
||||
return result
|
||||
|
||||
|
||||
def _build_report_fallback(query: str, context_text: str, source_count: int) -> str:
|
||||
def _fallback_outline_specs(
|
||||
custom_outline: str | None,
|
||||
*,
|
||||
structure_mode: str,
|
||||
) -> list[tuple[int, str, str]]:
|
||||
"""Keep every fixed heading, or only the adaptive outline's core levels."""
|
||||
|
||||
specs = _outline_heading_specs(custom_outline)
|
||||
if structure_mode == "fixed" or len(specs) < 2:
|
||||
return specs
|
||||
|
||||
# In adaptive mode, preserve the document title plus the major chapter
|
||||
# level. Subsections may legitimately be merged when evidence is thin.
|
||||
first, *remaining = specs
|
||||
first_is_title = (
|
||||
_normalized_heading_title(first[1]) in _OUTLINE_TITLE_PLACEHOLDERS
|
||||
or (remaining and first[0] < min(level for level, _title, _line in remaining))
|
||||
)
|
||||
chapters = remaining if first_is_title else specs
|
||||
if not chapters:
|
||||
return specs
|
||||
major_level = min(level for level, _title, _line in chapters)
|
||||
selected = [spec for spec in chapters if spec[0] == major_level]
|
||||
return ([first] if first_is_title else []) + selected
|
||||
|
||||
|
||||
def _build_outline_fallback(
|
||||
query: str,
|
||||
context_text: str,
|
||||
source_count: int,
|
||||
*,
|
||||
custom_outline: str,
|
||||
structure_mode: str,
|
||||
) -> str | None:
|
||||
"""Create a conservative report that still honors the selected structure."""
|
||||
|
||||
specs = _fallback_outline_specs(custom_outline, structure_mode=structure_mode)
|
||||
if not specs:
|
||||
return None
|
||||
|
||||
bounded_context = (context_text or "").strip()
|
||||
if len(bounded_context) > 6000:
|
||||
bounded_context = bounded_context[:6000].rstrip() + "\n\n(其余已确认材料未在保底正文中展开。)"
|
||||
quoted_context = "\n".join(f"> {line}" if line else ">" for line in bounded_context.splitlines())
|
||||
context_written = False
|
||||
output: list[str] = []
|
||||
for index, (level, title, markdown_line) in enumerate(specs):
|
||||
normalized_title = _normalized_heading_title(title)
|
||||
if normalized_title in _OUTLINE_TITLE_PLACEHOLDERS:
|
||||
output.append(f"{'#' * level} {query}")
|
||||
continue
|
||||
|
||||
output.append(markdown_line)
|
||||
if any(marker in title for marker in ("参考资料", "参考来源", "参考文献")):
|
||||
output.append(
|
||||
"- 本次未收集到可直接列示的外部来源;后续补充资料时应逐项核验并更新本节。"
|
||||
if source_count <= 0
|
||||
else "- 已确认材料保留在本次研究记录中;自动撰写未完成,需在后续重写时补齐逐项来源。"
|
||||
)
|
||||
continue
|
||||
|
||||
if source_count > 0 and bounded_context and not context_written:
|
||||
output.append(
|
||||
f"本次研究已收集到 {source_count} 条来源。由于自动撰写服务未能按所选结构完成成稿,"
|
||||
"现先保留所选模板并附上已确认材料摘录,供继续核验和重写。\n\n"
|
||||
f"{quoted_context}"
|
||||
)
|
||||
context_written = True
|
||||
continue
|
||||
|
||||
if source_count <= 0 and index == 0:
|
||||
output.append(
|
||||
f"本方案围绕“{query}”按所选模板保留完整结构。当前没有可引用的外部研究材料,"
|
||||
"本节仅给出待核验的推进框架,不把缺失事实写成确定结论。"
|
||||
)
|
||||
else:
|
||||
output.append(
|
||||
"现有材料不足以形成可靠的专项结论。本节按所选模板保留,待补充可核验事实、"
|
||||
"预算、责任主体和时间要求后完善;不得为填满结构而编造。"
|
||||
)
|
||||
|
||||
return "\n\n".join(output).strip() + "\n"
|
||||
|
||||
|
||||
def _build_report_fallback(
|
||||
query: str,
|
||||
context_text: str,
|
||||
source_count: int,
|
||||
*,
|
||||
custom_outline: str | None = None,
|
||||
structure_mode: str = "adaptive",
|
||||
) -> str:
|
||||
"""Build a non-refusal last-resort report from confirmed context.
|
||||
|
||||
This is deliberately conservative: it preserves a bounded extract of the
|
||||
@@ -1222,6 +1791,17 @@ def _build_report_fallback(query: str, context_text: str, source_count: int) ->
|
||||
preliminary, rather than inventing facts when the configured model cannot
|
||||
complete either writing attempt.
|
||||
"""
|
||||
if custom_outline and (
|
||||
outlined := _build_outline_fallback(
|
||||
query,
|
||||
context_text,
|
||||
source_count,
|
||||
custom_outline=custom_outline,
|
||||
structure_mode=structure_mode,
|
||||
)
|
||||
):
|
||||
return outlined
|
||||
|
||||
if source_count <= 0:
|
||||
return f"""# {query}
|
||||
|
||||
|
||||
+10
-35
@@ -1,9 +1,8 @@
|
||||
"""Small, durable report-delta emitter shared by research runners.
|
||||
"""Live report-delta emitter shared by research runners.
|
||||
|
||||
The event store intentionally persists every emitted event. Sending one event
|
||||
for each provider token would therefore create excessive database writes and
|
||||
make reconnect replay expensive. This helper keeps the first visible delta
|
||||
immediate, then coalesces later deltas into bounded chunks.
|
||||
Writing must stay visible even when MySQL is slow. Provider tokens go to the
|
||||
SSE hub immediately. The completed ``report_markdown`` is stored on the session
|
||||
after the model finishes — never as mid-stream ``report_chunk`` rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,17 +31,16 @@ class ReportChunkEmitter:
|
||||
self._phase = phase
|
||||
self._chunk_size = chunk_size
|
||||
self._buffer = ""
|
||||
self._sent_first = False
|
||||
self._next_offset = 0
|
||||
self._persisted_offset = 0
|
||||
|
||||
async def add(self, delta: str) -> None:
|
||||
"""Add a provider delta, making the first one visible immediately."""
|
||||
"""Add a provider delta, making it visible immediately."""
|
||||
if not delta:
|
||||
return
|
||||
# The sandbox UI receives the native model delta immediately. This is
|
||||
# intentionally non-durable; ``report_chunk`` below remains the
|
||||
# bounded checkpoint used for reconnect replay.
|
||||
# Must not await event-store IO here. This callback runs inside
|
||||
# ``model.astream``; a hung append pauses HTTP reads and the writing
|
||||
# connection dies after the first paragraph on the production server.
|
||||
await self._events.publish_live(
|
||||
"report_delta",
|
||||
phase=self._phase,
|
||||
@@ -55,14 +53,6 @@ class ReportChunkEmitter:
|
||||
)
|
||||
self._next_offset += len(delta)
|
||||
self._buffer += delta
|
||||
if not self._sent_first:
|
||||
first = self._buffer[: self._chunk_size]
|
||||
await self._emit(first)
|
||||
self._buffer = self._buffer[self._chunk_size :]
|
||||
self._sent_first = True
|
||||
while len(self._buffer) >= self._chunk_size:
|
||||
await self._emit(self._buffer[: self._chunk_size])
|
||||
self._buffer = self._buffer[self._chunk_size :]
|
||||
|
||||
async def add_thinking(self, delta: str) -> None:
|
||||
"""Forward reasoning to the live UI without persisting it as report text."""
|
||||
@@ -79,23 +69,8 @@ class ReportChunkEmitter:
|
||||
)
|
||||
|
||||
async def flush(self) -> None:
|
||||
"""Persist the final tail once the model stream has finished."""
|
||||
if self._buffer:
|
||||
await self._emit(self._buffer)
|
||||
self._buffer = ""
|
||||
|
||||
async def _emit(self, delta: str) -> None:
|
||||
await self._events.emit(
|
||||
"report_chunk",
|
||||
phase=self._phase,
|
||||
payload={
|
||||
"delta": delta,
|
||||
"target": self._target,
|
||||
"reportVersion": self._report_version,
|
||||
"offset": self._persisted_offset,
|
||||
},
|
||||
)
|
||||
self._persisted_offset += len(delta)
|
||||
"""Drop the in-memory buffer. Persistence happens after the report exists."""
|
||||
self._buffer = ""
|
||||
|
||||
|
||||
__all__ = ["ReportChunkEmitter"]
|
||||
Reference in new issue
Block a user