feat(backend): 完善 Workflow 与 DeepResearch 的运行链路与持久化能力

功能说明:\n- 重构并增强工作流网关执行链路,支持更完整的 proposal 与 run 调度参数透传\n- 增加 DeepResearch/workflow 运行事件持久化能力并补齐建表 SQL 与查询/保存接口适配\n- 优化 lead agent 与 deep research runner 的中间态处理,增强线程收集与执行链并发控制\n- 调整工作流执行器与任务路由,提升兼容性并完善错误与回放行为\n- 补充回归测试:deep_research 持久化、workflow 并发、定义加载、引擎执行、run 流程等覆盖
This commit is contained in:
董宏凯 committed 2026-09-02 12:52:37 +08:00
1 parent f5bf7897b9
commit 6d326a21cd
28 files changed
+836 -135

No files matched your search

@@ -45,6 +45,10 @@ logger = logging.getLogger(__name__)
# Upper bound on harvested materials so a chatty collector thread cannot blow
# up the report context; the curating/compressing stages trim further anyway.
MAX_HARVESTED_MATERIALS = 80
# A collector model may time out after tool calls were checkpointed but before
# its final assistant response advanced the thread head. Keep the scan bounded
# while retaining the richest checkpoint from that short history.
MAX_CHECKPOINT_HISTORY = 32
# Tool-name → ResearchMaterial.source_type. Everything unknown maps to
# ``other``; ``bash`` is treated as ``skill`` because skill scripts emit there.
@@ -72,11 +76,43 @@ async def harvest_thread_materials(
visible text (its collection小结) and doubles as a fallback material when
the thread produced no harvestable tool results.
"""
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": runtime_thread_id, "checkpoint_ns": ""}})
config = {"configurable": {"thread_id": runtime_thread_id, "checkpoint_ns": ""}}
checkpoint_tuple = await checkpointer.aget_tuple(config)
if checkpoint_tuple is None:
logger.warning("deep-research harvest: no checkpoint for thread %s", runtime_thread_id)
return [], ""
values = checkpoint_tuple.checkpoint.get("channel_values", {}) or {}
best_materials, best_summary = _harvest_checkpoint_tuple(checkpoint_tuple, query=query)
# ``aget_tuple`` normally returns the latest state. When a streamed lead
# agent call fails, however, some checkpointers leave that head at the
# final input checkpoint even though an earlier checkpoint already holds
# the successful web-search tool results. Prefer the richest available
# material pool rather than treating that transient head as an empty run.
checkpoint_history = getattr(checkpointer, "alist", None)
if callable(checkpoint_history):
try:
history = checkpoint_history(config, limit=MAX_CHECKPOINT_HISTORY)
if hasattr(history, "__aiter__"):
async for candidate in history:
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
if len(materials) > len(best_materials):
best_materials, best_summary = materials, summary
else:
for candidate in history:
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
if len(materials) > len(best_materials):
best_materials, best_summary = materials, summary
except Exception: # noqa: BLE001 - newest checkpoint remains a safe fallback
logger.debug("deep-research harvest: checkpoint history unavailable for %s", runtime_thread_id, exc_info=True)
return best_materials, best_summary
def _harvest_checkpoint_tuple(checkpoint_tuple: Any, *, query: str) -> tuple[list[ResearchMaterial], str]:
checkpoint = getattr(checkpoint_tuple, "checkpoint", None)
if not isinstance(checkpoint, dict):
return [], ""
values = checkpoint.get("channel_values", {}) or {}
messages = values.get("messages") or []
return harvest_messages_materials(messages, query=query)
@@ -259,6 +295,7 @@ class StaticHarvestMaterialProvider:
__all__ = [
"MAX_CHECKPOINT_HISTORY",
"MAX_HARVESTED_MATERIALS",
"StaticHarvestMaterialProvider",
"harvest_messages_materials",
@@ -120,6 +120,11 @@ class BasicRunner:
all_materials: list[ResearchMaterial] = []
plan: dict[str, Any] = {}
# A completed Deep Research request must still produce a useful report
# when every retrieval channel is temporarily unavailable. This flag
# carries that runtime fact into the writer without mutating the
# immutable per-job config snapshot.
use_knowledge_fill = False
try:
# ── 1. planning: generate sub-queries ──────────────────────────
@@ -198,32 +203,20 @@ class BasicRunner:
cancellation.raise_if_cancelled()
if not all_materials:
knowledge_fill = _allows_knowledge_fill(request.config.report_instruction)
# Collection is best-effort. In particular, a collector run
# can time out after its tool calls have streamed to the user,
# or every configured retrieval backend can be unavailable.
# Neither case should turn a report request into a stub file.
use_knowledge_fill = True
await events.emit(
"warning",
phase="curating",
payload={
"code": "no_materials",
"message": (
"未收集到外部研究材料,改用领域知识撰写"
if knowledge_fill
else "未收集到任何研究材料"
),
"recoverable": knowledge_fill,
"message": "未收集到外部研究材料,已改用领域知识继续撰写完整报告",
"recoverable": True,
},
)
if not knowledge_fill:
report_md = (
f"# {request.query}\n\n"
f"未能收集到与研究课题「{request.query}」相关的材料,无法生成报告。"
)
await adapters.artifacts.write_text("report.md", report_md)
await events.emit(
"report_completed",
phase="writing",
payload={"reportVersion": 1, "sourceCount": 0},
)
return DeepResearchResult(report_markdown=report_md, source_ids=[], plan=plan)
deduped: list[ResearchMaterial] = []
selected_ids: list[str] = []
context_list: list[dict[str, Any]] = []
@@ -261,7 +254,14 @@ class BasicRunner:
if cancellation:
cancellation.raise_if_cancelled()
report_md = await self._generate_report(llm, request, context_text, events, cancellation)
report_md = await self._generate_report(
llm,
request,
context_text,
events,
cancellation,
allow_knowledge_fill=use_knowledge_fill,
)
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
# ── 6. write artifacts ─────────────────────────────────────────
@@ -384,14 +384,23 @@ class BasicRunner:
return cleaned or queries_from_retrieval_directions(query, dirs)[:n]
async def _search_one(self, provider, query, config, request, cancellation, events) -> list[ResearchMaterial]:
"""Search one sub-query, emitting events. Callers drive the ordering."""
"""Search one sub-query while keeping evidence independent of telemetry.
A material batch is the report's input, whereas ``source_added`` is
only progress/persistence telemetry. In particular, a source may
contain native ``datetime`` metadata that an older event store cannot
serialise. Do not let that failure discard an already-returned batch.
"""
if cancellation:
cancellation.raise_if_cancelled()
await events.emit(
"query_started",
phase="collecting",
payload={"query": query},
)
try:
await events.emit(
"query_started",
phase="collecting",
payload={"query": query},
)
except Exception: # noqa: BLE001 - telemetry must not prevent collection
logger.warning("basic runner: unable to emit query_started", exc_info=True)
materials = await provider.search(
query,
domains=config.allowed_domains or None,
@@ -402,27 +411,42 @@ class BasicRunner:
)
# Emit source_added for each material.
for m in materials:
try:
await events.emit(
"source_added",
phase="collecting",
payload={
"sourceId": m.id,
"title": m.title,
"url": m.url,
"sourceType": m.source_type,
# Full persistence-ready dict so the executor can store
# sources without re-deriving them from the event.
"source": m.to_persistence_dict(query),
},
)
except Exception: # noqa: BLE001 - the batch still belongs in the report
logger.warning("basic runner: unable to emit source_added for %s", m.id, exc_info=True)
try:
await events.emit(
"source_added",
"query_completed",
phase="collecting",
payload={
"sourceId": m.id,
"title": m.title,
"url": m.url,
"sourceType": m.source_type,
# Full persistence-ready dict so the executor can store
# sources without re-deriving them from the event.
"source": m.to_persistence_dict(query),
},
payload={"query": query, "resultCount": len(materials)},
)
await events.emit(
"query_completed",
phase="collecting",
payload={"query": query, "resultCount": len(materials)},
)
except Exception: # noqa: BLE001 - preserve collected materials on telemetry failure
logger.warning("basic runner: unable to emit query_completed", exc_info=True)
return materials
async def _generate_report(self, llm, request: DeepResearchRequest, context_text: str, events, cancellation=None) -> str:
async def _generate_report(
self,
llm,
request: DeepResearchRequest,
context_text: str,
events,
cancellation=None,
*,
allow_knowledge_fill: bool = False,
) -> str:
"""Generate the final Markdown report from the formatted context."""
outline = request.config.custom_outline
filled_preamble = fill_report_preamble(
@@ -431,7 +455,7 @@ class BasicRunner:
)
outline_block = _report_outline_instruction(outline)
requirements_block = _report_requirements_instruction(request.config.report_instruction)
if _allows_knowledge_fill(request.config.report_instruction):
if allow_knowledge_fill or _allows_knowledge_fill(request.config.report_instruction):
system_extra = (
"用户已要求在材料不足时用模型自身的公开领域知识写成完整报告。"
"禁止把「材料不足/未提供文献」写成章节主线;参考文献可标注为基于公开领域知识。"
@@ -595,7 +619,7 @@ def fill_report_preamble(
continue
if re.match(r"^密级\s*[::]", stripped):
value = re.sub(r"^密级\s*[::]\s*", "", stripped)
filled.append(f"密级:内部" if _is_placeholder(value) else f"密级:{value.strip() or '内部'}")
filled.append("密级:内部" if _is_placeholder(value) else f"密级:{value.strip() or '内部'}")
continue
title = _PLACEHOLDER_RE.sub(topic, stripped) if topic else stripped
filled.append(title)
@@ -842,6 +842,16 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
is_scheduled_run=bool(cfg.get("is_scheduled_run", False)),
)
# 前端可提供当前视觉明暗模式给页面生成类任务。把值放入系统提示而不是
# 指望模型从 ToolRuntime.context 猜测,确保自定义 SOUL 也能稳定使用。
presentation_theme = str(cfg.get("presentation_theme") or "").strip().lower()
if presentation_theme in {"dark", "light"}:
system_prompt += (
"\n\n【当前界面主题】\n"
f"当前页面呈现主题为 `{presentation_theme}`。若本轮生成汇报页面,请优先选择同色系 HTML 模板;"
"除非用户明确要求覆盖该主题。"
)
system_prompt = _append_orchestration_capability_boundary(system_prompt, agent_id)
_log_es_query_routing_registration(