feat(deep-research): 强化任务链路可靠性并接入 agentfx 报告助手种子

功能说明:\n- 重构 Deep Research 调度与执行链,补齐 dispatcher、executor、live hub 与依赖注入入口,提升任务下发、执行、回调的稳定性\n- 增强运行时持久化与恢复能力,接入业务映射、任务来源与任务事件日志模型,覆盖持久化与执行可靠性关键路径测试\n- 新增 agentfx 报告助手 seed 与任务报告导入脚本配置,支持离线环境快速初始化与任务流模板回填\n- 完善 deep-research 与 thread_runs 路由参数编排,兼容不同业务映射与运行入口的前后端联动\n- 补充前端 agentfx deep-link 解析,保证 deep-research 场景下的登录跳转与任务上下文一致
This commit is contained in:
董宏凯 committed 2026-09-04 23:18:52 +08:00
1 parent 438772f0b7
commit 5c75e1af16
36 files changed
+3163 -312

No files matched your search

@@ -61,9 +61,10 @@ logger = logging.getLogger(__name__)
# ── constants ───────────────────────────────────────────────────────────────
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
EXECUTOR_REVISION = "deep-research-browser-db-checkpoint-20260903-v6"
_LEASE_TTL_SECONDS = 60
EXECUTOR_REVISION = "deep-research-db-first-client-fallback-20260904-v13"
_SOURCE_READ_ATTEMPTS = 16
_SOURCE_READ_DELAY_SECONDS = 0.5
_LEASE_TTL_SECONDS = 180
_HEARTBEAT_INTERVAL = 20.0
_CANCEL_POLL_INTERVAL = 5.0
_REPORT_PROJECTION_RETRY_ATTEMPTS = 3
@@ -96,10 +97,14 @@ def _terminal_report_validation_error(report: str | None) -> str | None:
return "empty_report"
headings = re.findall(r"(?m)^#{1,6}\s+\S+", text)
# A real report may quote or discuss a refusal/material-gap sentence in a
# limitations section. Only replace short or unstructured refusal output;
# a substantive multi-section report is accepted unchanged.
# limitations section. Only replace output that is *both* short *and*
# unstructured. The previous ``or`` rejected the zero-source fallback
# (five headings, but compact < 800 and a 局限性 sentence matching
# 没有…来源…不能), which is how “支持无素材写作” still looked like a
# failed write after session_read_repair.
if (
(len(compact) < 800 or len(headings) < 2)
len(compact) < 800
and len(headings) < 2
and any(pattern.search(compact) for pattern in _TERMINAL_REPORT_REFUSAL_PATTERNS)
):
return "refusal_semantics"
@@ -118,14 +123,21 @@ def _materials_from_browser_snapshot(
for value in values or []:
if not isinstance(value, dict):
continue
title = str(value.get("title") or "").strip()[:500]
url = str(value.get("url") or "").strip()[:2000] or None
content = str(value.get("raw_content") or value.get("snippet") or "").strip()[:6000]
title = str(value.get("title") or value.get("name") or "").strip()[:500]
url = str(value.get("url") or value.get("source_url") or value.get("link") or "").strip()[:2000] or None
content = str(
value.get("raw_content")
or value.get("content")
or value.get("content_preview")
or value.get("snippet")
or value.get("text")
or ""
).strip()[:6000]
if not title and not url and not content:
continue
raw_key = str(value.get("id") or value.get("rec_uuid") or url or f"{title}\n{content[:240]}")
digest = hashlib.sha256(raw_key.encode("utf-8", errors="ignore")).hexdigest()
material_id = str(value.get("id") or f"src_{digest[:16]}")[:128]
material_id = str(value.get("id") or f"src_{digest[:16]}")[:64]
if material_id in seen:
continue
seen.add(material_id)
@@ -149,6 +161,7 @@ def _materials_from_browser_snapshot(
break
return materials
# Phase → progress% mapping for the job timeline.
_PHASE_PROGRESS: dict[str, int] = {
"initializing": 0,
@@ -166,11 +179,29 @@ _PHASE_PROGRESS: dict[str, int] = {
}
_RECOVERABLE_SESSION_CODES = {
"no_materials",
"no_materials_write_fallback",
"chat_harvest_empty",
"empty_client_materials",
"force_no_materials",
"browser_snapshot_received",
"client_materials_loaded_from_source_table",
"material_channel_fallback_used",
"client_materials_reloaded_before_write",
"report_template_fallback",
"report_stream_retry",
"report_refusal_retry",
}
def _session_diagnostic_error(diagnostics: list[dict[str, Any]]) -> str | None:
"""Render real degraded-run causes into the session API's error field."""
lines: list[str] = []
for item in diagnostics[-20:]:
code = str(item.get("code") or "deep_research_degraded")
if code in _RECOVERABLE_SESSION_CODES:
continue
stage = str(item.get("stage") or "unknown")
error_type = str(item.get("errorType") or item.get("outcome") or "Warning")
detail = str(item.get("detail") or "").strip()
@@ -314,6 +345,7 @@ class DeepResearchJobExecutor:
event_store: DeepResearchEventRepository,
source_store: DeepResearchSourceRepository | None = None,
live_publisher: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
live_subscriber_probe: Callable[[str], bool] | None = None,
checkpointer: Any | None = None,
document_rewrite_version_store: Any | None = None,
app_config: AppConfig | None = None,
@@ -323,6 +355,10 @@ class DeepResearchJobExecutor:
self._event_store = event_store
self._source_store = source_store
self._live_publisher = live_publisher
# Tells the event sink whether this worker actually holds the viewer's
# SSE connection. Live-only reasoning frames are mirrored into the
# durable log when it does not (multi-worker deployments).
self._live_subscriber_probe = live_subscriber_probe
# Needed by the chat entry to read the collector thread's messages.
self._checkpointer = checkpointer
self._document_rewrite_version_store = document_rewrite_version_store
@@ -423,6 +459,31 @@ class DeepResearchJobExecutor:
entry = str(snapshot.get("entry") or "legacy")
snapshot_projection_diagnostics: list[dict[str, Any]] = []
if entry == "chat" and self._job_store is not None:
try:
latest_job = await self._job_store.get_unscoped(job_id)
latest_snapshot = (latest_job or {}).get("input_snapshot")
if isinstance(latest_snapshot, dict):
snapshot = {**snapshot, **latest_snapshot}
entry = str(snapshot.get("entry") or entry)
query = str(snapshot.get("query") or query)
title = str(snapshot.get("title") or title)
except Exception: # noqa: BLE001 - claimed snapshot remains usable
logger.warning(
"deep-research job %s could not reload the latest input snapshot",
job_id,
exc_info=True,
)
expected_material_count = 0
if entry == "chat":
raw_expected = snapshot.get("collector_material_count")
try:
expected_material_count = int(raw_expected or 0)
except (TypeError, ValueError):
expected_material_count = 0
snapshot_list = snapshot.get("collector_materials")
if expected_material_count <= 0 and isinstance(snapshot_list, list):
expected_material_count = len(snapshot_list)
# A structural regeneration has its own cloned durable source pool.
# The browser snapshot is intentionally kept frozen in the job
# input until the final fallback step. Persisting it here would
@@ -439,7 +500,17 @@ class DeepResearchJobExecutor:
lease_owner=lease_owner,
cancel_token=cancel_token,
entry=entry,
collector_materials=(snapshot.get("collector_materials") or []) if entry == "chat" else None,
collector_materials=(
snapshot.get("collector_materials")
if isinstance(snapshot.get("collector_materials"), list)
else []
)
if entry == "chat"
else None,
materials_authoritative=bool(snapshot.get("client_materials_authoritative"))
and entry == "chat",
expected_material_count=expected_material_count,
force_no_materials=bool(snapshot.get("force_no_materials")) and entry == "chat",
)
except (ResearchAwaitingInput, ResearchCancelled):
raise
@@ -463,7 +534,15 @@ class DeepResearchJobExecutor:
)
except Exception as snapshot_exc: # noqa: BLE001
snapshot_read_error = snapshot_exc
if not snapshot_materials and self._source_store is not None:
if (
not snapshot_materials
and self._source_store is not None
and not bool(snapshot.get("force_no_materials"))
and not (
bool(snapshot.get("client_materials_authoritative"))
and expected_material_count <= 0
)
):
try:
rows = await self._source_store.list_by_session(
session_id,
@@ -527,6 +606,25 @@ class DeepResearchJobExecutor:
or snapshot_read_error.__class__.__name__,
}
)
# Make the degradation visible in the run trace. A job that
# reports `completed / progress 100 / errorCode null` while
# actually returning a template report is indistinguishable
# from a healthy run, which is exactly what made this class of
# failure so hard to diagnose on one deployment.
with suppress(Exception):
await self._emit_warning(
job_id,
session_id,
payload={
"code": "execution_failed_report_fallback",
"message": "报告生成链路异常,已根据已收集素材生成保底报告;素材未丢失,可点击「选择结构再生成」重试。",
"recoverable": True,
"errorType": exc.__class__.__name__,
"detail": str(exc)[:1200] or exc.__class__.__name__,
"diagnostic": diagnostics[0],
"diagnostics": diagnostics,
},
)
result = DeepResearchResult(
report_markdown=_build_report_fallback(
query,
@@ -564,7 +662,25 @@ class DeepResearchJobExecutor:
except asyncio.CancelledError:
logger.info("deep-research job %s cancelled (task)", job_id)
raise
with suppress(Exception):
await self._job_store.finalize_cancel(job_id)
if session_id:
with suppress(Exception):
row = await self._session_store.get(session_id, user_id=user_id)
if row is not None and row.get("status") in ("running", "awaiting_input"):
status = (
"awaiting_report"
if row.get("runtime_thread_id")
or (row.get("config_snapshot") or {}).get("collection_mode") == "chat"
else "cancelled"
)
await self._session_store.update(
session_id,
user_id=user_id,
status=status,
active_job_id=None,
)
return
except Exception as exc: # noqa: BLE001
logger.exception("deep-research job %s crashed", job_id)
@@ -1563,6 +1679,9 @@ class DeepResearchJobExecutor:
cancel_token: CancellationToken,
entry: str = "legacy",
collector_materials: list[dict[str, Any]] | None = None,
materials_authoritative: bool = False,
expected_material_count: int = 0,
force_no_materials: bool = False,
) -> DeepResearchResult:
from deerflow.agents.deep_research.adapters.artifacts import ThreadOutputsArtifactWriter
from deerflow.agents.deep_research.adapters.context import DeerFlowContextCompressor
@@ -1580,6 +1699,7 @@ class DeepResearchJobExecutor:
session_id=session_id,
job_id=job_id,
on_event=self._live_publisher,
live_subscriber_probe=self._live_subscriber_probe,
)
recording_sink = RecordingEventSink(
base_sink,
@@ -1594,19 +1714,47 @@ class DeepResearchJobExecutor:
# LLM backend (single chokepoint).
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
# Material provider. The chat entry serves the pool already harvested
# from the collector-agent conversation; legacy keeps the search tiers.
# Material provider. First write prefers the source table (full pool
# persisted when collection finished). The job snapshot only carries a
# small browser fallback. 「选择结构再生成」 reads the same table.
if entry == "chat":
material_provider = await self._build_chat_materials(
job_id=job_id,
session_id=session_id,
user_id=user_id,
runtime_thread_id=runtime_thread_id,
query=query,
events=recording_sink,
cancellation=cancel_token,
collector_materials=collector_materials,
from deerflow.agents.deep_research.collection.thread_harvester import (
StaticHarvestMaterialProvider,
)
if force_no_materials:
material_provider = StaticHarvestMaterialProvider([])
try:
await recording_sink.emit(
"warning",
phase="collecting",
payload={
"code": "force_no_materials",
"message": "用户勾选了无素材写作,已忽略素材表中的已收集资料",
"recoverable": True,
},
)
await recording_sink.emit(
"collection_completed",
phase="collecting",
payload={"source": "force_no_materials", "materialCount": 0},
)
except Exception: # noqa: BLE001 - writing still uses the empty pool
logger.warning("failed to publish force-no-materials diagnostic", exc_info=True)
else:
material_provider = await self._build_chat_materials(
job_id=job_id,
session_id=session_id,
user_id=user_id,
runtime_thread_id=runtime_thread_id,
query=query,
events=recording_sink,
cancellation=cancel_token,
collector_materials=collector_materials,
materials_authoritative=materials_authoritative,
expected_material_count=expected_material_count,
force_no_materials=False,
)
elif entry == "regenerate":
material_provider = await self._build_selected_materials(
session_id=session_id,
@@ -1748,8 +1896,16 @@ class DeepResearchJobExecutor:
events,
cancellation: CancellationToken,
collector_materials: list[dict[str, Any]] | None = None,
materials_authoritative: bool = False,
expected_material_count: int = 0,
force_no_materials: bool = False,
):
"""Harvest the collector thread into a static material pool.
"""Build the first-write material pool.
Prefer the source table (the full collected pool). The browser list is
a small fallback used only when that table is empty. Checkpoint harvest
is last. An authoritative empty list (count 0) is a true no-materials
write and must not pick leftover source rows.
Emits ``collection_completed`` plus one ``source_added`` per material so
the sources panel fills exactly like the legacy channel (the recording
@@ -1771,6 +1927,12 @@ class DeepResearchJobExecutor:
material_channel = "none"
browser_snapshot_persisted = False
channel_failures: list[dict[str, str]] = []
snapshot_count = len(collector_materials) if isinstance(collector_materials, list) else 0
if expected_material_count <= 0:
expected_material_count = snapshot_count
skip_leftover_sources = force_no_materials or (
materials_authoritative and expected_material_count <= 0
)
def persisted_rows_to_materials(rows: list[dict[str, Any]]) -> list[ResearchMaterial]:
allowed_types = {"web", "knowledge", "skill", "mcp", "file", "other"}
@@ -1799,6 +1961,42 @@ class DeepResearchJobExecutor:
async def try_browser_snapshot() -> None:
nonlocal browser_material_count, browser_snapshot_persisted, harvested, material_channel
raw_count = snapshot_count
if raw_count == 0 and expected_material_count > 0:
# Older jobs stored only a count here; blobs live in the table.
return
sample_keys = (
sorted({str(key) for item in (collector_materials or [])[:3] if isinstance(item, dict) for key in item})
if raw_count
else []
)
try:
await events.emit(
"warning",
phase="collecting",
payload={
"code": "browser_snapshot_received",
"message": (
f"后端已收到前端素材快照 {raw_count} 条"
if raw_count
else (
"后端未收到前端素材快照(字段缺失或空数组)。"
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
)
),
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
"receivedType": type(collector_materials).__name__,
"receivedCount": raw_count,
"expectedCount": expected_material_count,
"sampleKeys": sample_keys[:20],
},
},
)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish browser snapshot receipt", exc_info=True)
if not collector_materials:
channel_failures.append(
{
@@ -1834,6 +2032,7 @@ class DeepResearchJobExecutor:
session_id=session_id,
job_id=job_id,
user_id=user_id or None,
selected=True,
**material.to_persistence_dict(material.query),
)
except Exception as exc: # noqa: BLE001 - report can still use the snapshot
@@ -1865,53 +2064,8 @@ class DeepResearchJobExecutor:
exc_info=True,
)
# First-report recovery is deliberately sequential, not additive:
# browser snapshot → durable source rows → runtime checkpoint. Once a
# channel provides material, later channels are not opened.
await try_browser_snapshot()
if not harvested:
try:
if self._checkpointer is None:
raise RuntimeError("Deep Research checkpointer is unavailable")
if not runtime_thread_id:
raise RuntimeError("Deep Research collector thread is unavailable")
checkpoint_materials, checkpoint_summary = await harvest_thread_materials(
checkpointer=self._checkpointer,
runtime_thread_id=runtime_thread_id,
query=query,
)
checkpoint_material_count = len(checkpoint_materials)
agent_summary = checkpoint_summary or ""
harvested = dedupe_materials(checkpoint_materials)[:MAX_HARVESTED_MATERIALS]
if harvested:
material_channel = "runtime_checkpoint"
else:
channel_failures.append(
{
"channel": "runtime_checkpoint",
"outcome": "empty",
"detail": "运行时会话未返回可用素材",
}
)
except ResearchCancelled:
raise
except Exception as exc: # noqa: BLE001 - all earlier channels remain available
channel_failures.append(
{
"channel": "runtime_checkpoint",
"outcome": "error",
"errorType": exc.__class__.__name__,
"detail": str(exc)[:1200] or exc.__class__.__name__,
}
)
logger.warning(
"deep-research collector checkpoint harvest failed; no earlier material channel was usable "
"(thread=%s, error=%s)",
runtime_thread_id,
exc.__class__.__name__,
)
if not harvested:
async def try_durable_sources(*, retry: bool) -> None:
nonlocal durable_material_count, harvested, material_channel
if self._source_store is None:
channel_failures.append(
{
@@ -1920,7 +2074,11 @@ class DeepResearchJobExecutor:
"detail": "素材存储库不可用",
}
)
else:
return
attempts = _SOURCE_READ_ATTEMPTS if retry else 1
last_error: Exception | None = None
for attempt in range(attempts):
cancellation.raise_if_cancelled()
try:
rows = await self._source_store.list_by_session(
session_id,
@@ -1934,27 +2092,109 @@ class DeepResearchJobExecutor:
harvested = dedupe_materials(persisted_materials)[:MAX_HARVESTED_MATERIALS]
if harvested:
material_channel = "durable_sources"
try:
await events.emit(
"warning",
phase="collecting",
payload={
"code": "client_materials_loaded_from_source_table",
"message": f"已从素材表读取 {len(harvested)} 条写作材料",
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "durable_sources",
"durableMaterialCount": durable_material_count,
"expectedCount": expected_material_count,
"attempt": attempt + 1,
},
},
)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish durable source diagnostic", exc_info=True)
return
except Exception as exc: # noqa: BLE001 - retry or enter no-material
last_error = exc
logger.warning(
"chat deep-research durable material read failed (attempt %s/%s)",
attempt + 1,
attempts,
exc_info=True,
)
if attempt + 1 < attempts:
await asyncio.sleep(_SOURCE_READ_DELAY_SECONDS)
if last_error is not None:
channel_failures.append(
{
"channel": "durable_sources",
"outcome": "error",
"errorType": last_error.__class__.__name__,
"detail": str(last_error)[:1200] or last_error.__class__.__name__,
}
)
else:
channel_failures.append(
{
"channel": "durable_sources",
"outcome": "empty",
"detail": "数据库中没有可用素材",
}
)
if skip_leftover_sources:
await try_browser_snapshot()
if not harvested:
channel_failures.append(
{
"channel": "client_materials",
"outcome": "empty",
"detail": "前端传入素材为空,已进入无素材写作",
}
)
else:
await try_durable_sources(retry=True)
if not harvested:
await try_browser_snapshot()
if not harvested:
try:
if self._checkpointer is None:
raise RuntimeError("Deep Research checkpointer is unavailable")
if not runtime_thread_id:
raise RuntimeError("Deep Research collector thread is unavailable")
checkpoint_materials, checkpoint_summary = await harvest_thread_materials(
checkpointer=self._checkpointer,
runtime_thread_id=runtime_thread_id,
query=query,
)
checkpoint_material_count = len(checkpoint_materials)
agent_summary = checkpoint_summary or ""
harvested = dedupe_materials(checkpoint_materials)[:MAX_HARVESTED_MATERIALS]
if harvested:
material_channel = "runtime_checkpoint"
else:
channel_failures.append(
{
"channel": "durable_sources",
"channel": "runtime_checkpoint",
"outcome": "empty",
"detail": "数据库中没有可用素材",
"detail": "运行时会话未返回可用素材",
}
)
except Exception as exc: # noqa: BLE001 - no-material report remains valid
except ResearchCancelled:
raise
except Exception as exc: # noqa: BLE001 - all earlier channels remain available
channel_failures.append(
{
"channel": "durable_sources",
"channel": "runtime_checkpoint",
"outcome": "error",
"errorType": exc.__class__.__name__,
"detail": str(exc)[:1200] or exc.__class__.__name__,
}
)
logger.warning(
"chat deep-research durable material read failed; entering no-material route if needed",
exc_info=True,
)
logger.warning(
"deep-research collector checkpoint harvest failed; no earlier material channel was usable "
"(thread=%s, error=%s)",
runtime_thread_id,
exc.__class__.__name__,
)
# Freeze the harvested list before publishing any progress. Source
# event persistence is intentionally best-effort: a bad payload or a
@@ -1971,6 +2211,7 @@ class DeepResearchJobExecutor:
"browserMaterialCount": browser_material_count,
"checkpointMaterialCount": checkpoint_material_count,
"durableMaterialCount": durable_material_count,
"expectedMaterialCount": expected_material_count,
"agentSummaryChars": len(agent_summary),
},
)
@@ -1998,7 +2239,30 @@ class DeepResearchJobExecutor:
material.id,
exc_info=True,
)
if material_provider.material_count and material_channel != "browser_snapshot":
if material_provider.material_count and material_channel == "browser_snapshot":
try:
await events.emit(
"warning",
phase="collecting",
payload={
"code": "material_channel_fallback_used",
"message": "素材表为空,已使用前端兜底素材撰写",
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "client_material_fallback",
"selectedChannel": material_channel,
"browserMaterialCount": browser_material_count,
"checkpointMaterialCount": checkpoint_material_count,
"durableMaterialCount": durable_material_count,
"runtimeThreadId": runtime_thread_id,
"failures": channel_failures,
},
},
)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
elif material_provider.material_count and material_channel == "runtime_checkpoint":
try:
await events.emit(
"warning",
@@ -2021,7 +2285,7 @@ class DeepResearchJobExecutor:
)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
if not material_provider.material_count:
if not material_provider.material_count and not force_no_materials:
try:
await events.emit(
"warning",
@@ -2034,6 +2298,11 @@ class DeepResearchJobExecutor:
"stage": "collecting",
"outcome": "all_material_channels_empty",
"runtimeThreadId": runtime_thread_id,
"receivedSnapshotCount": (
len(collector_materials) if isinstance(collector_materials, list) else 0
),
"expectedMaterialCount": expected_material_count,
"receivedSnapshotType": type(collector_materials).__name__,
"failures": channel_failures,
},
},
@@ -2297,6 +2566,25 @@ class DeepResearchJobExecutor:
with suppress(Exception): # noqa: BLE001
await self._emit_terminal(job_id, session_id, "job_failed")
async def _emit_warning(
self,
job_id: str,
session_id: str,
*,
payload: dict[str, Any],
phase: str = "writing",
) -> None:
"""Record one durable, operator-readable degradation notice."""
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
sink = PersistingEventSink(
self._event_store,
session_id=session_id,
job_id=job_id,
on_event=self._live_publisher,
)
await sink.emit("warning", phase=phase, payload=payload)
async def _emit_terminal(self, job_id: str, session_id: str, event_type: str) -> None:
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink