feat(backend): 完善 Workflow 与 DeepResearch 的运行链路与持久化能力
功能说明:\n- 重构并增强工作流网关执行链路,支持更完整的 proposal 与 run 调度参数透传\n- 增加 DeepResearch/workflow 运行事件持久化能力并补齐建表 SQL 与查询/保存接口适配\n- 优化 lead agent 与 deep research runner 的中间态处理,增强线程收集与执行链并发控制\n- 调整工作流执行器与任务路由,提升兼容性并完善错误与回放行为\n- 补充回归测试:deep_research 持久化、workflow 并发、定义加载、引擎执行、run 流程等覆盖
This commit is contained in:
1 parent
f5bf7897b9
commit
6d326a21cd
28 files changed
+836
-135
No files matched your search
+39
-2
@@ -45,6 +45,10 @@ logger = logging.getLogger(__name__)
|
||||
# Upper bound on harvested materials so a chatty collector thread cannot blow
|
||||
# up the report context; the curating/compressing stages trim further anyway.
|
||||
MAX_HARVESTED_MATERIALS = 80
|
||||
# A collector model may time out after tool calls were checkpointed but before
|
||||
# its final assistant response advanced the thread head. Keep the scan bounded
|
||||
# while retaining the richest checkpoint from that short history.
|
||||
MAX_CHECKPOINT_HISTORY = 32
|
||||
|
||||
# Tool-name → ResearchMaterial.source_type. Everything unknown maps to
|
||||
# ``other``; ``bash`` is treated as ``skill`` because skill scripts emit there.
|
||||
@@ -72,11 +76,43 @@ async def harvest_thread_materials(
|
||||
visible text (its collection小结) and doubles as a fallback material when
|
||||
the thread produced no harvestable tool results.
|
||||
"""
|
||||
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": runtime_thread_id, "checkpoint_ns": ""}})
|
||||
config = {"configurable": {"thread_id": runtime_thread_id, "checkpoint_ns": ""}}
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(config)
|
||||
if checkpoint_tuple is None:
|
||||
logger.warning("deep-research harvest: no checkpoint for thread %s", runtime_thread_id)
|
||||
return [], ""
|
||||
values = checkpoint_tuple.checkpoint.get("channel_values", {}) or {}
|
||||
|
||||
best_materials, best_summary = _harvest_checkpoint_tuple(checkpoint_tuple, query=query)
|
||||
|
||||
# ``aget_tuple`` normally returns the latest state. When a streamed lead
|
||||
# agent call fails, however, some checkpointers leave that head at the
|
||||
# final input checkpoint even though an earlier checkpoint already holds
|
||||
# the successful web-search tool results. Prefer the richest available
|
||||
# material pool rather than treating that transient head as an empty run.
|
||||
checkpoint_history = getattr(checkpointer, "alist", None)
|
||||
if callable(checkpoint_history):
|
||||
try:
|
||||
history = checkpoint_history(config, limit=MAX_CHECKPOINT_HISTORY)
|
||||
if hasattr(history, "__aiter__"):
|
||||
async for candidate in history:
|
||||
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
|
||||
if len(materials) > len(best_materials):
|
||||
best_materials, best_summary = materials, summary
|
||||
else:
|
||||
for candidate in history:
|
||||
materials, summary = _harvest_checkpoint_tuple(candidate, query=query)
|
||||
if len(materials) > len(best_materials):
|
||||
best_materials, best_summary = materials, summary
|
||||
except Exception: # noqa: BLE001 - newest checkpoint remains a safe fallback
|
||||
logger.debug("deep-research harvest: checkpoint history unavailable for %s", runtime_thread_id, exc_info=True)
|
||||
return best_materials, best_summary
|
||||
|
||||
|
||||
def _harvest_checkpoint_tuple(checkpoint_tuple: Any, *, query: str) -> tuple[list[ResearchMaterial], str]:
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", None)
|
||||
if not isinstance(checkpoint, dict):
|
||||
return [], ""
|
||||
values = checkpoint.get("channel_values", {}) or {}
|
||||
messages = values.get("messages") or []
|
||||
return harvest_messages_materials(messages, query=query)
|
||||
|
||||
@@ -259,6 +295,7 @@ class StaticHarvestMaterialProvider:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_CHECKPOINT_HISTORY",
|
||||
"MAX_HARVESTED_MATERIALS",
|
||||
"StaticHarvestMaterialProvider",
|
||||
"harvest_messages_materials",
|
||||
|
||||
+68
-44
@@ -120,6 +120,11 @@ class BasicRunner:
|
||||
|
||||
all_materials: list[ResearchMaterial] = []
|
||||
plan: dict[str, Any] = {}
|
||||
# A completed Deep Research request must still produce a useful report
|
||||
# when every retrieval channel is temporarily unavailable. This flag
|
||||
# carries that runtime fact into the writer without mutating the
|
||||
# immutable per-job config snapshot.
|
||||
use_knowledge_fill = False
|
||||
|
||||
try:
|
||||
# ── 1. planning: generate sub-queries ──────────────────────────
|
||||
@@ -198,32 +203,20 @@ class BasicRunner:
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
if not all_materials:
|
||||
knowledge_fill = _allows_knowledge_fill(request.config.report_instruction)
|
||||
# Collection is best-effort. In particular, a collector run
|
||||
# can time out after its tool calls have streamed to the user,
|
||||
# or every configured retrieval backend can be unavailable.
|
||||
# Neither case should turn a report request into a stub file.
|
||||
use_knowledge_fill = True
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="curating",
|
||||
payload={
|
||||
"code": "no_materials",
|
||||
"message": (
|
||||
"未收集到外部研究材料,改用领域知识撰写"
|
||||
if knowledge_fill
|
||||
else "未收集到任何研究材料"
|
||||
),
|
||||
"recoverable": knowledge_fill,
|
||||
"message": "未收集到外部研究材料,已改用领域知识继续撰写完整报告",
|
||||
"recoverable": True,
|
||||
},
|
||||
)
|
||||
if not knowledge_fill:
|
||||
report_md = (
|
||||
f"# {request.query}\n\n"
|
||||
f"未能收集到与研究课题「{request.query}」相关的材料,无法生成报告。"
|
||||
)
|
||||
await adapters.artifacts.write_text("report.md", report_md)
|
||||
await events.emit(
|
||||
"report_completed",
|
||||
phase="writing",
|
||||
payload={"reportVersion": 1, "sourceCount": 0},
|
||||
)
|
||||
return DeepResearchResult(report_markdown=report_md, source_ids=[], plan=plan)
|
||||
deduped: list[ResearchMaterial] = []
|
||||
selected_ids: list[str] = []
|
||||
context_list: list[dict[str, Any]] = []
|
||||
@@ -261,7 +254,14 @@ class BasicRunner:
|
||||
if cancellation:
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
report_md = await self._generate_report(llm, request, context_text, events, cancellation)
|
||||
report_md = await self._generate_report(
|
||||
llm,
|
||||
request,
|
||||
context_text,
|
||||
events,
|
||||
cancellation,
|
||||
allow_knowledge_fill=use_knowledge_fill,
|
||||
)
|
||||
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
|
||||
|
||||
# ── 6. write artifacts ─────────────────────────────────────────
|
||||
@@ -384,14 +384,23 @@ class BasicRunner:
|
||||
return cleaned or queries_from_retrieval_directions(query, dirs)[:n]
|
||||
|
||||
async def _search_one(self, provider, query, config, request, cancellation, events) -> list[ResearchMaterial]:
|
||||
"""Search one sub-query, emitting events. Callers drive the ordering."""
|
||||
"""Search one sub-query while keeping evidence independent of telemetry.
|
||||
|
||||
A material batch is the report's input, whereas ``source_added`` is
|
||||
only progress/persistence telemetry. In particular, a source may
|
||||
contain native ``datetime`` metadata that an older event store cannot
|
||||
serialise. Do not let that failure discard an already-returned batch.
|
||||
"""
|
||||
if cancellation:
|
||||
cancellation.raise_if_cancelled()
|
||||
await events.emit(
|
||||
"query_started",
|
||||
phase="collecting",
|
||||
payload={"query": query},
|
||||
)
|
||||
try:
|
||||
await events.emit(
|
||||
"query_started",
|
||||
phase="collecting",
|
||||
payload={"query": query},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - telemetry must not prevent collection
|
||||
logger.warning("basic runner: unable to emit query_started", exc_info=True)
|
||||
materials = await provider.search(
|
||||
query,
|
||||
domains=config.allowed_domains or None,
|
||||
@@ -402,27 +411,42 @@ class BasicRunner:
|
||||
)
|
||||
# Emit source_added for each material.
|
||||
for m in materials:
|
||||
try:
|
||||
await events.emit(
|
||||
"source_added",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"sourceId": m.id,
|
||||
"title": m.title,
|
||||
"url": m.url,
|
||||
"sourceType": m.source_type,
|
||||
# Full persistence-ready dict so the executor can store
|
||||
# sources without re-deriving them from the event.
|
||||
"source": m.to_persistence_dict(query),
|
||||
},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - the batch still belongs in the report
|
||||
logger.warning("basic runner: unable to emit source_added for %s", m.id, exc_info=True)
|
||||
try:
|
||||
await events.emit(
|
||||
"source_added",
|
||||
"query_completed",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"sourceId": m.id,
|
||||
"title": m.title,
|
||||
"url": m.url,
|
||||
"sourceType": m.source_type,
|
||||
# Full persistence-ready dict so the executor can store
|
||||
# sources without re-deriving them from the event.
|
||||
"source": m.to_persistence_dict(query),
|
||||
},
|
||||
payload={"query": query, "resultCount": len(materials)},
|
||||
)
|
||||
await events.emit(
|
||||
"query_completed",
|
||||
phase="collecting",
|
||||
payload={"query": query, "resultCount": len(materials)},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - preserve collected materials on telemetry failure
|
||||
logger.warning("basic runner: unable to emit query_completed", exc_info=True)
|
||||
return materials
|
||||
|
||||
async def _generate_report(self, llm, request: DeepResearchRequest, context_text: str, events, cancellation=None) -> str:
|
||||
async def _generate_report(
|
||||
self,
|
||||
llm,
|
||||
request: DeepResearchRequest,
|
||||
context_text: str,
|
||||
events,
|
||||
cancellation=None,
|
||||
*,
|
||||
allow_knowledge_fill: bool = False,
|
||||
) -> str:
|
||||
"""Generate the final Markdown report from the formatted context."""
|
||||
outline = request.config.custom_outline
|
||||
filled_preamble = fill_report_preamble(
|
||||
@@ -431,7 +455,7 @@ class BasicRunner:
|
||||
)
|
||||
outline_block = _report_outline_instruction(outline)
|
||||
requirements_block = _report_requirements_instruction(request.config.report_instruction)
|
||||
if _allows_knowledge_fill(request.config.report_instruction):
|
||||
if allow_knowledge_fill or _allows_knowledge_fill(request.config.report_instruction):
|
||||
system_extra = (
|
||||
"用户已要求在材料不足时用模型自身的公开领域知识写成完整报告。"
|
||||
"禁止把「材料不足/未提供文献」写成章节主线;参考文献可标注为基于公开领域知识。"
|
||||
@@ -595,7 +619,7 @@ def fill_report_preamble(
|
||||
continue
|
||||
if re.match(r"^密级\s*[::]", stripped):
|
||||
value = re.sub(r"^密级\s*[::]\s*", "", stripped)
|
||||
filled.append(f"密级:内部" if _is_placeholder(value) else f"密级:{value.strip() or '内部'}")
|
||||
filled.append("密级:内部" if _is_placeholder(value) else f"密级:{value.strip() or '内部'}")
|
||||
continue
|
||||
title = _PLACEHOLDER_RE.sub(topic, stripped) if topic else stripped
|
||||
filled.append(title)
|
||||
|
||||
@@ -842,6 +842,16 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
is_scheduled_run=bool(cfg.get("is_scheduled_run", False)),
|
||||
)
|
||||
|
||||
# 前端可提供当前视觉明暗模式给页面生成类任务。把值放入系统提示而不是
|
||||
# 指望模型从 ToolRuntime.context 猜测,确保自定义 SOUL 也能稳定使用。
|
||||
presentation_theme = str(cfg.get("presentation_theme") or "").strip().lower()
|
||||
if presentation_theme in {"dark", "light"}:
|
||||
system_prompt += (
|
||||
"\n\n【当前界面主题】\n"
|
||||
f"当前页面呈现主题为 `{presentation_theme}`。若本轮生成汇报页面,请优先选择同色系 HTML 模板;"
|
||||
"除非用户明确要求覆盖该主题。"
|
||||
)
|
||||
|
||||
system_prompt = _append_orchestration_capability_boundary(system_prompt, agent_id)
|
||||
|
||||
_log_es_query_routing_registration(
|
||||
|
||||
Reference in new issue
Block a user