feat(deep-research): 强化任务链路可靠性并接入 agentfx 报告助手种子
功能说明:\n- 重构 Deep Research 调度与执行链,补齐 dispatcher、executor、live hub 与依赖注入入口,提升任务下发、执行、回调的稳定性\n- 增强运行时持久化与恢复能力,接入业务映射、任务来源与任务事件日志模型,覆盖持久化与执行可靠性关键路径测试\n- 新增 agentfx 报告助手 seed 与任务报告导入脚本配置,支持离线环境快速初始化与任务流模板回填\n- 完善 deep-research 与 thread_runs 路由参数编排,兼容不同业务映射与运行入口的前后端联动\n- 补充前端 agentfx deep-link 解析,保证 deep-research 场景下的登录跳转与任务上下文一致
This commit is contained in:
1 parent
438772f0b7
commit
5c75e1af16
36 files changed
+3163
-312
No files matched your search
+108
@@ -16,8 +16,10 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.agents.deep_research.events import (
|
||||
@@ -42,6 +44,20 @@ def _json_default(value: Any) -> Any:
|
||||
return str(value)
|
||||
|
||||
|
||||
# Live-only frame types that have no durable counterpart at all. Report and
|
||||
# summary text are already checkpointed (``report_chunk`` / ``summary_completed``),
|
||||
# so only the reasoning channel needs a cross-worker mirror.
|
||||
_MIRRORED_LIVE_TYPES = frozenset({"report_thinking", "plan_reasoning", "summary_thinking"})
|
||||
# Coalescing bounds for that mirror: one bounded row per ~600 characters or
|
||||
# ~1.5 seconds, never one row per provider token.
|
||||
_MIRROR_MIN_CHARS = 600
|
||||
_MIRROR_MAX_DELAY_S = 1.5
|
||||
# Floor for the time-based flush: a slowly trickling reasoning stream must not
|
||||
# degrade into one durable row per token. Anything shorter is carried until the
|
||||
# next durable milestone flushes it.
|
||||
_MIRROR_MIN_FLUSH_CHARS = 40
|
||||
|
||||
|
||||
def _json_safe_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize event payloads before they reach any event repository.
|
||||
|
||||
@@ -66,11 +82,18 @@ class PersistingEventSink:
|
||||
session_id: str,
|
||||
job_id: str,
|
||||
on_event: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
live_subscriber_probe: Callable[[str], bool] | None = None,
|
||||
) -> None:
|
||||
self._repo = repo
|
||||
self._session_id = session_id
|
||||
self._job_id = job_id
|
||||
self._on_event = on_event
|
||||
# Reports whether this worker holds an SSE connection for the job.
|
||||
# Absent probe → assume the live path reaches the viewer (single-worker
|
||||
# deployments and tests keep today's zero-extra-write behaviour).
|
||||
self._live_subscriber_probe = live_subscriber_probe
|
||||
self._mirror_buffers: dict[str, dict[str, Any]] = {}
|
||||
self._mirroring = False
|
||||
self.diagnostics: list[dict[str, Any]] = []
|
||||
# A durable event row normally owns the sequence. When the event
|
||||
# database is temporarily unavailable, use distinct negative sequence
|
||||
@@ -87,6 +110,10 @@ class PersistingEventSink:
|
||||
) -> DeepResearchEvent:
|
||||
event_payload = _json_safe_payload(payload or {})
|
||||
persistence_error: Exception | None = None
|
||||
if not self._mirroring:
|
||||
# Order any coalesced reasoning tail before the next milestone
|
||||
# instead of leaving it buffered until the job ends.
|
||||
await self._flush_live_mirrors()
|
||||
try:
|
||||
row = await self._repo.append(
|
||||
session_id=self._session_id,
|
||||
@@ -211,6 +238,87 @@ class PersistingEventSink:
|
||||
type,
|
||||
exc_info=True,
|
||||
)
|
||||
with suppress(Exception):
|
||||
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
|
||||
|
||||
# ── cross-worker mirror for live-only reasoning frames ──────────────────
|
||||
|
||||
async def _mirror_live_frame(
|
||||
self,
|
||||
type: str, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase,
|
||||
payload: dict[str, Any],
|
||||
) -> None:
|
||||
"""Buffer a live-only reasoning delta when no local viewer can see it.
|
||||
|
||||
The live hub is in-process. A job runs on the worker that won its
|
||||
durable lease, while the SSE connection is served by whichever worker
|
||||
accepted it, so on a multi-worker deployment the model's thinking is
|
||||
usually published into a hub with no subscribers and is then lost —
|
||||
there is no durable row to replay it from. When that is the case, the
|
||||
reasoning is coalesced here and persisted as a bounded durable event
|
||||
so the polling stream on the other worker still renders it.
|
||||
"""
|
||||
if type not in _MIRRORED_LIVE_TYPES or self._live_subscriber_probe is None:
|
||||
return
|
||||
delta = payload.get("delta")
|
||||
if not isinstance(delta, str) or not delta:
|
||||
return
|
||||
if self._live_subscriber_probe(self._job_id):
|
||||
# A viewer is attached here and has already received every frame
|
||||
# live. Drop the buffer rather than flushing it, so that viewer
|
||||
# does not also receive the same text as a durable replay.
|
||||
self._mirror_buffers.pop(self._mirror_key(type, payload), None)
|
||||
return
|
||||
key = self._mirror_key(type, payload)
|
||||
buffered = self._mirror_buffers.get(key)
|
||||
if buffered is None:
|
||||
buffered = {
|
||||
"type": type,
|
||||
"phase": phase,
|
||||
"target": payload.get("target"),
|
||||
"reportVersion": payload.get("reportVersion"),
|
||||
"text": "",
|
||||
"since": monotonic(),
|
||||
}
|
||||
self._mirror_buffers[key] = buffered
|
||||
buffered["text"] += delta
|
||||
buffered["phase"] = phase
|
||||
buffered_chars = len(buffered["text"])
|
||||
aged_out = (
|
||||
buffered_chars >= _MIRROR_MIN_FLUSH_CHARS
|
||||
and monotonic() - float(buffered["since"]) >= _MIRROR_MAX_DELAY_S
|
||||
)
|
||||
if buffered_chars >= _MIRROR_MIN_CHARS or aged_out:
|
||||
await self._emit_mirror(key)
|
||||
|
||||
@staticmethod
|
||||
def _mirror_key(type: str, payload: dict[str, Any]) -> str: # noqa: A002
|
||||
return f"{type}::{payload.get('target') or 'report'}"
|
||||
|
||||
async def _flush_live_mirrors(self) -> None:
|
||||
for key in tuple(self._mirror_buffers):
|
||||
with suppress(Exception):
|
||||
await self._emit_mirror(key)
|
||||
|
||||
async def _emit_mirror(self, key: str) -> None:
|
||||
buffered = self._mirror_buffers.pop(key, None)
|
||||
if buffered is None or not buffered["text"]:
|
||||
return
|
||||
payload: dict[str, Any] = {
|
||||
"delta": buffered["text"],
|
||||
"liveMirror": True,
|
||||
}
|
||||
if buffered["target"] is not None:
|
||||
payload["target"] = buffered["target"]
|
||||
if buffered["reportVersion"] is not None:
|
||||
payload["reportVersion"] = buffered["reportVersion"]
|
||||
self._mirroring = True
|
||||
try:
|
||||
await self.emit(buffered["type"], phase=buffered["phase"], payload=payload)
|
||||
finally:
|
||||
self._mirroring = False
|
||||
|
||||
|
||||
class NullEventSink:
|
||||
|
||||
+55
-26
@@ -33,7 +33,7 @@ from deerflow.agents.deep_research.cancellation import CancellationToken
|
||||
from deerflow.agents.deep_research.config import DeepResearchConfig, ModelRole
|
||||
from deerflow.agents.deep_research.events import EventSink
|
||||
from deerflow.agents.deep_research.types import CompletionResult
|
||||
from deerflow.utils.stream_text import InlineThinkTagFilter
|
||||
from deerflow.utils.stream_text import InlineThinkTagFilter, strip_inline_think_tags
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain.chat_models import BaseChatModel
|
||||
@@ -388,31 +388,57 @@ class DeerFlowCompletionBackend:
|
||||
# Custom ``report_delta`` events remain the only live preview channel.
|
||||
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
|
||||
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
|
||||
attempts = 2 if operation.startswith("write_report") else 1
|
||||
last_error: Exception | None = None
|
||||
try:
|
||||
async for chunk in model.astream(
|
||||
to_langchain_messages(messages),
|
||||
config=isolated_config,
|
||||
**kwargs,
|
||||
):
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
reasoning = _reasoning_delta(chunk)
|
||||
if reasoning and on_reasoning is not None:
|
||||
await on_reasoning(reasoning)
|
||||
# A few OpenAI-compatible models put their reasoning in the
|
||||
# ordinary content channel instead of ``reasoning_content``.
|
||||
# Split it from Markdown even when tags span tokens: callers may
|
||||
# display a live thought trace, but it never joins ``text``.
|
||||
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
|
||||
if inline_reasoning and on_reasoning is not None:
|
||||
await on_reasoning(inline_reasoning)
|
||||
if delta:
|
||||
text_parts.append(delta)
|
||||
if on_delta is not None:
|
||||
await on_delta(delta)
|
||||
chunk_usage = _extract_usage(chunk)
|
||||
if any(chunk_usage.values()):
|
||||
usage = chunk_usage
|
||||
for attempt in range(attempts):
|
||||
text_parts.clear()
|
||||
inline_think_filter = InlineThinkTagFilter()
|
||||
try:
|
||||
async for chunk in model.astream(
|
||||
to_langchain_messages(messages),
|
||||
config=isolated_config,
|
||||
**kwargs,
|
||||
):
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
reasoning = _reasoning_delta(chunk)
|
||||
if reasoning and on_reasoning is not None:
|
||||
await on_reasoning(reasoning)
|
||||
# A few OpenAI-compatible models put their reasoning in the
|
||||
# ordinary content channel instead of ``reasoning_content``.
|
||||
# Split it from Markdown even when tags span tokens: callers may
|
||||
# display a live thought trace, but it never joins ``text``.
|
||||
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
|
||||
if inline_reasoning and on_reasoning is not None:
|
||||
await on_reasoning(inline_reasoning)
|
||||
if delta:
|
||||
text_parts.append(delta)
|
||||
if on_delta is not None:
|
||||
await on_delta(delta)
|
||||
chunk_usage = _extract_usage(chunk)
|
||||
if any(chunk_usage.values()):
|
||||
usage = chunk_usage
|
||||
last_error = None
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001 - report writing waits like ordinary chat
|
||||
from deerflow.agents.deep_research.cancellation import ResearchCancelled
|
||||
|
||||
if isinstance(exc, (ResearchCancelled, asyncio.CancelledError)):
|
||||
raise
|
||||
last_error = exc
|
||||
# Only replay a clean connection; a half-written stream is
|
||||
# reset by the runner, not by appending a second attempt.
|
||||
if text_parts or attempt + 1 >= attempts:
|
||||
raise
|
||||
logger.warning(
|
||||
"deep_research llm: %s stream interrupted before the first token (%s); retrying once",
|
||||
operation,
|
||||
type(exc).__name__,
|
||||
)
|
||||
await asyncio.sleep(1.5)
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
finally:
|
||||
reset_parent_config()
|
||||
|
||||
@@ -470,7 +496,10 @@ class DeerFlowCompletionBackend:
|
||||
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
text = _content_to_text(response)
|
||||
# ``ainvoke`` returns raw content, including the inline ``<think>``
|
||||
# block some OpenAI-compatible deployments write there. The streamed
|
||||
# path filters it out, so this retry path must not reintroduce it.
|
||||
text = strip_inline_think_tags(_content_to_text(response))
|
||||
usage = _extract_usage(response)
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
return CompletionResult(text=text, model=model_name, usage=usage)
|
||||
|
||||
+7
@@ -80,6 +80,13 @@ EventType = Literal[
|
||||
"deep_branch_completed",
|
||||
"report_reset",
|
||||
"report_chunk",
|
||||
# Model reasoning normally travels as a live-only frame. It becomes a
|
||||
# durable event when no SSE connection for the job is attached to the
|
||||
# worker running it, because the in-process live hub cannot reach a
|
||||
# viewer served by a different worker (see PersistingEventSink).
|
||||
"report_thinking",
|
||||
"plan_reasoning",
|
||||
"summary_thinking",
|
||||
"report_completed",
|
||||
"summary_completed",
|
||||
"collection_completed",
|
||||
|
||||
+240
-48
@@ -22,6 +22,7 @@ vendored prompt families in a later commit without changing the runner shape.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
@@ -46,6 +47,23 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Delta size used to replay a non-streamed retry report. Matches the durable
|
||||
# chunk size so the retry produces the same event volume as a streamed write.
|
||||
_RETRY_REPLAY_CHUNK = 480
|
||||
_WRITE_REPORT_TIMEOUT_SECONDS = 180.0
|
||||
|
||||
|
||||
async def _await_writer(coro):
|
||||
"""Fail a hung production stream before the job lease is reclaimed."""
|
||||
try:
|
||||
return await asyncio.wait_for(coro, timeout=_WRITE_REPORT_TIMEOUT_SECONDS)
|
||||
except TimeoutError as exc:
|
||||
raise CompletionError(
|
||||
f"Report writer exceeded {_WRITE_REPORT_TIMEOUT_SECONDS:g} seconds",
|
||||
code="write_report_timeout",
|
||||
retryable=True,
|
||||
) from exc
|
||||
|
||||
|
||||
# ── structured-output schemas ───────────────────────────────────────────────
|
||||
|
||||
@@ -508,11 +526,12 @@ class BasicRunner:
|
||||
*,
|
||||
collection_outcomes: list[dict[str, Any]] | None = None,
|
||||
) -> DeepResearchResult:
|
||||
"""Write a complete fallback report when collection returned no rows.
|
||||
"""Write a complete report when collection returned no rows.
|
||||
|
||||
Research collection is best-effort. A collector timeout must not be
|
||||
persisted as a "cannot generate" pseudo-report, regardless of which
|
||||
runner mode is active.
|
||||
Stream the same model writer as a normal report so the sandbox stays
|
||||
empty through thinking and fills token-by-token. A refusal or transport
|
||||
failure still falls through ``_generate_report`` to the deterministic
|
||||
template; do not dump that template before the model has written.
|
||||
"""
|
||||
events = adapters.events
|
||||
await events.emit(
|
||||
@@ -520,7 +539,7 @@ class BasicRunner:
|
||||
phase="curating",
|
||||
payload={
|
||||
"code": "no_materials",
|
||||
"message": "未收集到外部研究材料,已改用领域知识继续撰写完整报告",
|
||||
"message": "未收集到外部研究材料,将按公开领域知识撰写完整报告",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
@@ -529,30 +548,55 @@ class BasicRunner:
|
||||
},
|
||||
},
|
||||
)
|
||||
await events.emit(
|
||||
"report_reset",
|
||||
phase="writing",
|
||||
payload={"reportVersion": 1, "reason": "no_materials_write"},
|
||||
)
|
||||
await events.emit(
|
||||
"phase_changed",
|
||||
phase="writing",
|
||||
payload={"from": "curating", "to": "writing"},
|
||||
)
|
||||
context_text = (
|
||||
"本次没有可引用的外部研究材料。请只依据公开领域常识,就课题写成一份"
|
||||
"完整、审慎的分析报告。不要拒绝生成,也不要把「没有资料」写成全文主线;"
|
||||
"在局限性中说明证据边界即可。"
|
||||
)
|
||||
report_md = await self._generate_report(
|
||||
adapters.llm,
|
||||
request,
|
||||
_KNOWLEDGE_FILL_EMPTY_CONTEXT,
|
||||
context_text,
|
||||
events,
|
||||
cancellation,
|
||||
allow_knowledge_fill=True,
|
||||
source_count=0,
|
||||
)
|
||||
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
|
||||
await adapters.artifacts.write_text("report.md", report_md)
|
||||
await adapters.artifacts.write_text("sources.json", "[]")
|
||||
await adapters.artifacts.write_text("research-plan.json", json.dumps(plan, ensure_ascii=False, indent=2))
|
||||
await events.emit("report_completed", phase="writing", payload={"reportVersion": 1, "sourceCount": 0})
|
||||
await events.emit(
|
||||
"artifact_created",
|
||||
phase="exporting",
|
||||
payload={"format": "markdown", "name": "report.md"},
|
||||
)
|
||||
image_paths: dict[str, str] = {}
|
||||
try:
|
||||
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - illustrations must not discard the report
|
||||
logger.warning("no-materials image generation failed; retaining the text report", exc_info=True)
|
||||
for name, content in (
|
||||
("report.md", report_md),
|
||||
("sources.json", "[]"),
|
||||
("research-plan.json", json.dumps(plan, ensure_ascii=False, indent=2)),
|
||||
):
|
||||
try:
|
||||
await adapters.artifacts.write_text(name, content)
|
||||
except Exception: # noqa: BLE001 - session persistence still receives the report
|
||||
logger.warning("no-materials artifact write failed for %s", name, exc_info=True)
|
||||
try:
|
||||
await events.emit("report_completed", phase="writing", payload={"reportVersion": 1, "sourceCount": 0})
|
||||
await events.emit(
|
||||
"artifact_created",
|
||||
phase="exporting",
|
||||
payload={"format": "markdown", "name": "report.md"},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - completion events must not drop the report
|
||||
logger.warning("no-materials completion events failed; retaining the report", exc_info=True)
|
||||
return DeepResearchResult(
|
||||
report_markdown=report_md,
|
||||
source_ids=[],
|
||||
@@ -619,30 +663,110 @@ class BasicRunner:
|
||||
if request.config.structure_mode == "fixed" and (outline or "").strip():
|
||||
system += "撰写时必须原样沿用用户大纲中的章节标题与层级,不得改写成通用报告套式。"
|
||||
|
||||
completion_messages = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
|
||||
async def write_once(*, report_version: int) -> str:
|
||||
emitter = ReportChunkEmitter(events, report_version=report_version)
|
||||
if filled_preamble:
|
||||
await emitter.add(filled_preamble + "\n\n")
|
||||
completion_messages = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
result = await llm.stream_complete(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
operation="write_report",
|
||||
on_delta=emitter.add,
|
||||
on_reasoning=emitter.add_thinking,
|
||||
result = await _await_writer(
|
||||
llm.stream_complete(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
operation="write_report",
|
||||
on_delta=emitter.add,
|
||||
on_reasoning=emitter.add_thinking,
|
||||
)
|
||||
)
|
||||
await emitter.flush()
|
||||
return ensure_outline_preamble(result.text, outline, request.query)
|
||||
|
||||
async def retry_report(
|
||||
*,
|
||||
report_version: int,
|
||||
reason: str,
|
||||
code: str,
|
||||
message: str,
|
||||
error_type: str,
|
||||
error_detail: str,
|
||||
) -> str:
|
||||
"""Spend one real second model call before any deterministic report.
|
||||
|
||||
A broken ``astream`` says nothing about the model's ability to
|
||||
answer: deployments that stall past the inter-chunk deadline (idle
|
||||
proxy, long reasoning pause) still return a complete response.
|
||||
Dropping straight to the template silently downgrades a recoverable
|
||||
transport failure into a material dump that looks like a success.
|
||||
"""
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="writing",
|
||||
payload={
|
||||
"code": code,
|
||||
"message": message,
|
||||
"recoverable": True,
|
||||
"errorType": error_type,
|
||||
"diagnostic": {
|
||||
"stage": "writing",
|
||||
"outcome": reason,
|
||||
"errorType": error_type,
|
||||
"detail": error_detail[:1200] or error_type,
|
||||
},
|
||||
},
|
||||
)
|
||||
await events.emit(
|
||||
"report_reset",
|
||||
phase="writing",
|
||||
payload={"reportVersion": report_version, "reason": reason},
|
||||
)
|
||||
emitter = ReportChunkEmitter(events, report_version=report_version)
|
||||
# Prefer the non-streaming transport when the adapter offers it: a
|
||||
# stream that already died on an inter-chunk deadline usually dies
|
||||
# the same way again, while ``ainvoke`` waits for the whole answer.
|
||||
complete_prose = getattr(llm, "complete_prose", None)
|
||||
if callable(complete_prose):
|
||||
result = await _await_writer(
|
||||
complete_prose(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
operation="write_report_retry",
|
||||
)
|
||||
)
|
||||
report = ensure_outline_preamble(result.text, outline, request.query)
|
||||
# Replay as bounded deltas so the live sandbox and the durable
|
||||
# chunk log see the same shape as a streamed write.
|
||||
for index in range(0, len(report), _RETRY_REPLAY_CHUNK):
|
||||
await emitter.add(report[index : index + _RETRY_REPLAY_CHUNK])
|
||||
else:
|
||||
result = await _await_writer(
|
||||
llm.stream_complete(
|
||||
model_role="smart",
|
||||
messages=completion_messages,
|
||||
operation="write_report_retry",
|
||||
on_delta=emitter.add,
|
||||
on_reasoning=emitter.add_thinking,
|
||||
)
|
||||
)
|
||||
report = ensure_outline_preamble(result.text, outline, request.query)
|
||||
await emitter.flush()
|
||||
if not report.strip():
|
||||
raise CompletionError(
|
||||
"Report retry returned empty prose",
|
||||
code="empty_report_retry",
|
||||
retryable=False,
|
||||
)
|
||||
return report
|
||||
|
||||
async def complete_with_fallback(
|
||||
*,
|
||||
reason: str,
|
||||
message: str,
|
||||
error_type: str | None = None,
|
||||
error_detail: str | None = None,
|
||||
report_version: int = 2,
|
||||
) -> str:
|
||||
"""Replace an invalid streamed draft with a usable deterministic report.
|
||||
|
||||
@@ -668,39 +792,81 @@ class BasicRunner:
|
||||
await events.emit(
|
||||
"report_reset",
|
||||
phase="writing",
|
||||
payload={"reportVersion": 2, "reason": reason},
|
||||
payload={"reportVersion": report_version, "reason": reason},
|
||||
)
|
||||
fallback = ensure_outline_preamble(
|
||||
_build_report_fallback(request.query, context_text, source_count or 0),
|
||||
outline,
|
||||
request.query,
|
||||
)
|
||||
emitter = ReportChunkEmitter(events, report_version=2)
|
||||
emitter = ReportChunkEmitter(events, report_version=report_version)
|
||||
await emitter.add(fallback)
|
||||
await emitter.flush()
|
||||
return fallback
|
||||
|
||||
async def refusal_fallback() -> str:
|
||||
return await complete_with_fallback(
|
||||
reason="refusal_after_retry",
|
||||
message="模型两次返回拒绝文本,已根据可用材料生成保底报告",
|
||||
error_type="InvalidReportOutput",
|
||||
error_detail=report,
|
||||
report_version=3,
|
||||
)
|
||||
|
||||
try:
|
||||
report = await write_once(report_version=1)
|
||||
except Exception as exc: # noqa: BLE001 - preserve materials without a second model call
|
||||
if not _is_retryable_report_write_error(exc):
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - writing must finish even on a transport blip
|
||||
try:
|
||||
report = await retry_report(
|
||||
report_version=2,
|
||||
reason="transient_write_error",
|
||||
code="report_stream_retry",
|
||||
message="报告写入连接中断,正在重试生成(已收集素材保持不变)",
|
||||
error_type=type(exc).__name__,
|
||||
error_detail=str(exc),
|
||||
)
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
return await complete_with_fallback(
|
||||
reason="report_stream_error",
|
||||
message="报告写入连接中断,已根据可用材料生成保底报告",
|
||||
error_type=type(exc).__name__,
|
||||
error_detail=str(exc),
|
||||
)
|
||||
except Exception as retry_exc: # noqa: BLE001 - now use the deterministic report
|
||||
return await complete_with_fallback(
|
||||
reason="transient_write_error_after_retry",
|
||||
message="报告写入连接中断且重试未成功,已根据可用材料生成保底报告",
|
||||
error_type=type(retry_exc).__name__,
|
||||
error_detail=(
|
||||
f"stream={type(exc).__name__}: {str(exc)[:600]}; "
|
||||
f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}"
|
||||
),
|
||||
report_version=3,
|
||||
)
|
||||
return report if not _is_refusal_report(report) else await refusal_fallback()
|
||||
|
||||
if not _is_refusal_report(report):
|
||||
return report
|
||||
|
||||
return await complete_with_fallback(
|
||||
reason="refusal_output",
|
||||
message="模型返回了拒绝文本,已根据可用材料生成保底报告",
|
||||
error_type="InvalidReportOutput",
|
||||
error_detail=report,
|
||||
)
|
||||
# A refusal is a content failure, not a transport failure: one clean
|
||||
# rewrite recovers it far more often than a material dump does.
|
||||
try:
|
||||
report = await retry_report(
|
||||
report_version=2,
|
||||
reason="invalid_refusal_output",
|
||||
code="report_refusal_retry",
|
||||
message="模型返回了拒绝文本,正在重试生成报告",
|
||||
error_type="InvalidReportOutput",
|
||||
error_detail=report,
|
||||
)
|
||||
except ResearchCancelled:
|
||||
raise
|
||||
except Exception as retry_exc: # noqa: BLE001 - keep the confirmed materials
|
||||
return await complete_with_fallback(
|
||||
reason="refusal_output",
|
||||
message="模型返回了拒绝文本且重试失败,已根据可用材料生成保底报告",
|
||||
error_type=type(retry_exc).__name__,
|
||||
error_detail=f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}",
|
||||
report_version=3,
|
||||
)
|
||||
return report if not _is_refusal_report(report) else await refusal_fallback()
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────
|
||||
@@ -998,7 +1164,29 @@ def _is_retryable_report_write_error(exc: Exception) -> bool:
|
||||
if isinstance(exc, (TimeoutError, ConnectionError, OSError)):
|
||||
return True
|
||||
name = type(exc).__name__.lower()
|
||||
return any(token in name for token in ("timeout", "connection", "network", "streamchunk"))
|
||||
text = f"{name} {exc}".lower()
|
||||
return any(
|
||||
token in text
|
||||
for token in (
|
||||
"timeout",
|
||||
"connection",
|
||||
"network",
|
||||
"streamchunk",
|
||||
"readtimeout",
|
||||
"connecterror",
|
||||
"remoteprotocol",
|
||||
"chunkedencoding",
|
||||
"apiconnection",
|
||||
"apitimeout",
|
||||
"ratelimit",
|
||||
"unavailable",
|
||||
"badgateway",
|
||||
"gateway",
|
||||
"502",
|
||||
"503",
|
||||
"504",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _error_diagnostic(stage: str, exc: Exception, *, query: str | None = None) -> dict[str, str]:
|
||||
@@ -1039,23 +1227,27 @@ def _build_report_fallback(query: str, context_text: str, source_count: int) ->
|
||||
|
||||
## 摘要
|
||||
|
||||
本报告围绕“{query}”形成基础研究框架。本次外部检索未返回可引用来源,因此以下内容仅用于明确研究范围、分析维度与后续核验路径。
|
||||
本报告围绕“{query}”给出可继续推进的研究框架。外部检索通道本次没有返回可引用条目,正文依据公开领域常识整理分析维度,供后续核验使用。
|
||||
|
||||
## 研究范围与方法
|
||||
|
||||
围绕课题梳理核心对象、时间范围、影响因素和可验证指标;后续应补充权威公开数据、政策文件、原始访谈或业务台账进行交叉验证。
|
||||
围绕课题梳理核心对象、时间范围、影响因素和可验证指标。后续应补充权威公开数据、政策文件、原始访谈或业务台账,再对关键判断做交叉验证。
|
||||
|
||||
## 基础分析框架
|
||||
|
||||
可从现状与背景、关键驱动因素、风险与不确定性、利益相关方影响及可执行建议五个方面展开研判。所有具体事实、数值和结论均应在补充来源后确认。
|
||||
可从现状与背景、关键驱动因素、风险与不确定性、利益相关方影响及可执行建议五个方面展开研判。凡涉及具体事实、数值和因果判断,均应在补充来源后确认。
|
||||
|
||||
## 后续核验建议
|
||||
|
||||
优先补齐时间线、主体关系、公开统计口径和相互独立的原始记录。每一条新增材料都应对回本框架中的对应问题,避免只堆叠摘要。
|
||||
|
||||
## 局限性
|
||||
|
||||
本次没有外部来源可供引用,不能将本报告中的研究框架视为已完成的事实核验。
|
||||
本次成稿未附外部引用条目。文中框架用于界定问题,不等于已经完成事实核验。
|
||||
|
||||
## 结论
|
||||
|
||||
建议以本框架为基础继续补充可追溯材料,并在材料到位后更新各项实证结论。
|
||||
建议以本框架为基础继续补充可追溯材料,并在材料到位后更新各项实证结论。即使用户侧素材通道暂时不可用,研究报告也应先完成这一版可阅读成稿。
|
||||
"""
|
||||
|
||||
bounded_context = (context_text or "").strip()
|
||||
|
||||
+5
-5
@@ -1,10 +1,10 @@
|
||||
"""ORM model for the global business → agent / chain mapping (业务链条管理).
|
||||
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 7BF / 8BF) to an
|
||||
optional 智能体 (``agent_id``) and an optional 业务链条 (``chain_id`` → a
|
||||
``roundtable_chains`` row, which must be **public** so every account can resolve
|
||||
it). Used to auto-pick the business chain when a task page launches the
|
||||
multi-agent roundtable.
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
|
||||
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
|
||||
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
|
||||
there is no ``user_id`` column; all accounts see and use the same configuration.
|
||||
|
||||
+24
-3
@@ -1,7 +1,7 @@
|
||||
"""SQLAlchemy-backed global business → agent / chain mapping store.
|
||||
|
||||
Global / shared (no ``user_id`` scoping): every account reads and writes the same
|
||||
rows. The four business rows are seeded once on startup (``ensure_seed_defaults``)
|
||||
rows. Fixed business rows are seeded once on startup (``ensure_seed_defaults``)
|
||||
with empty agent/chain; users fill those in from the management page via
|
||||
``update_mapping``.
|
||||
"""
|
||||
@@ -21,12 +21,22 @@ _UNSET = object()
|
||||
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
|
||||
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
{"business_code": "7BF", "label": "XDFX"},
|
||||
{"business_code": "8BF", "label": "8BF"},
|
||||
{"business_code": "AFX", "label": "AGENTFX"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
{"business_code": "6-4", "label": "6-4"},
|
||||
{"business_code": "7-1", "label": "7-1"},
|
||||
{"business_code": "7-2", "label": "7-2"},
|
||||
{"business_code": "7-3", "label": "7-3"},
|
||||
{"business_code": "7-4", "label": "7-4"},
|
||||
]
|
||||
|
||||
|
||||
@@ -94,14 +104,24 @@ class BusinessMappingRepository:
|
||||
async def ensure_seed_defaults(self) -> int:
|
||||
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
|
||||
|
||||
Never overwrites an existing row (respects user-configured agent/chain).
|
||||
Never overwrites an existing row's agent/chain (respects user config).
|
||||
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
|
||||
the intended table order next to older seeded rows.
|
||||
"""
|
||||
inserted = 0
|
||||
dirty = False
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
|
||||
existing = await session.get(BusinessMappingRow, seed["business_code"])
|
||||
if existing is not None:
|
||||
if existing.sort_order != idx:
|
||||
existing.sort_order = idx
|
||||
dirty = True
|
||||
seed_label = str(seed.get("label") or "")
|
||||
if seed_label and not (existing.label or "").strip():
|
||||
existing.label = seed_label
|
||||
dirty = True
|
||||
continue
|
||||
session.add(
|
||||
BusinessMappingRow(
|
||||
@@ -117,6 +137,7 @@ class BusinessMappingRepository:
|
||||
)
|
||||
)
|
||||
inserted += 1
|
||||
if inserted:
|
||||
dirty = True
|
||||
if dirty:
|
||||
await session.commit()
|
||||
return inserted
|
||||
+35
-1
@@ -137,6 +137,38 @@ class DeepResearchJobRepository:
|
||||
# Should not happen in practice.
|
||||
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
|
||||
|
||||
async def replace_input_snapshot(self, job_id: str, input_snapshot: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Overwrite the frozen start payload on a still-queued job.
|
||||
|
||||
Idempotent retries used to keep the first snapshot forever. A first
|
||||
write that raced ahead of the browser evidence (or a reused empty
|
||||
job) then ran without ``collector_materials`` even though the later
|
||||
request carried them.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.status.in_(("queued", "running")),
|
||||
)
|
||||
.values(
|
||||
input_snapshot=_dumps(input_snapshot),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
@@ -198,12 +230,13 @@ class DeepResearchJobRepository:
|
||||
stmt = (
|
||||
select(DeepResearchJobRow.id)
|
||||
.where(
|
||||
DeepResearchJobRow.cancel_requested.is_(False),
|
||||
or_(
|
||||
DeepResearchJobRow.status == "queued",
|
||||
(DeepResearchJobRow.status == "running")
|
||||
& DeepResearchJobRow.lease_until.is_not(None)
|
||||
& (DeepResearchJobRow.lease_until < now),
|
||||
)
|
||||
),
|
||||
)
|
||||
.order_by(DeepResearchJobRow.updated_at.asc())
|
||||
.limit(limit)
|
||||
@@ -227,6 +260,7 @@ class DeepResearchJobRepository:
|
||||
}
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.cancel_requested.is_(False),
|
||||
or_(
|
||||
DeepResearchJobRow.status == "queued",
|
||||
(DeepResearchJobRow.status == "running")
|
||||
|
||||
+10
@@ -153,6 +153,16 @@ class DeepResearchSourceRepository:
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
source_rows = (await session.execute(stmt)).scalars().all()
|
||||
if not source_rows:
|
||||
# Collector persist marks rows selected, but an older session
|
||||
# may only have unselected harvest rows. Cloning nothing would
|
||||
# make 「选择结构再生成」 write with no materials.
|
||||
stmt = select(DeepResearchSourceRow).where(
|
||||
DeepResearchSourceRow.session_id == source_session_id,
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
source_rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
id_map: dict[str, str] = {}
|
||||
cloned_rows: list[DeepResearchSourceRow] = []
|
||||
|
||||
@@ -191,14 +191,14 @@ class RunManager:
|
||||
action: "interrupt" keeps checkpoint, "rollback" reverts to pre-run state.
|
||||
|
||||
Sets the abort event with the action reason and cancels the asyncio task.
|
||||
Returns ``True`` if the run was in-flight and cancellation was initiated.
|
||||
Returns ``True`` if cancellation was initiated or the run was already stopped.
|
||||
"""
|
||||
async with self._lock:
|
||||
record = self._runs.get(run_id)
|
||||
if record is None:
|
||||
return False
|
||||
return True
|
||||
if record.status not in (RunStatus.pending, RunStatus.running):
|
||||
return False
|
||||
return True
|
||||
record.abort_action = action
|
||||
record.abort_event.set()
|
||||
if record.task is not None and not record.task.done():
|
||||
|
||||
+96
-31
@@ -1389,17 +1389,40 @@ class ScheduledTaskService:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_conversion_prompt(skeleton_json: str, source: str, embedded_json: Any | None = None) -> str:
|
||||
def _build_conversion_prompt(
|
||||
skeleton_json: str,
|
||||
source: str,
|
||||
embedded_json: Any | None = None,
|
||||
instructions: str | None = None,
|
||||
) -> str:
|
||||
parts = [
|
||||
"请把下面的【源数据】转换为【目标结构】要求的 JSON。",
|
||||
"请把下面的【源数据】转换成【目标结构】要求的 JSON,用于填充一个 HTML 展示模板。",
|
||||
"模板只是一层**样式外壳**:它的字段名固定(页面脚本按字段名读取,绝不能改),"
|
||||
"但每个字段的**含义要按源数据的主题重新解释**——把主题内容语义映射进各分区,"
|
||||
"例如:总览指标填主题的核心数字;分组/区域类列表填主题的分区、品类或阵营;"
|
||||
"组织/单位类列表填主题的机构、厂商或层级体系;装备/条目类列表填主题的产品、"
|
||||
"技术或关键要素;事件/项目类列表填主题的时间线事项。内容必须全部来自【源数据】"
|
||||
"的主题,不要被模板原有示例的领域限制住。",
|
||||
"源数据可能是一段 JSON、一段纯文本、或包含 JSON 的文本——"
|
||||
"其中若有结构化 JSON 以 JSON 内容为准;若是纯文本请理解其语义后据实填充;"
|
||||
"源里没有对应内容的字段按其类型留空(\"\" / [] / null),不要臆造。",
|
||||
"若下面的【任务要求】里带有对页面的明确修改指示(如“标题改为…”“只更新某部分”),"
|
||||
"优先满足这些指示,其余字段可留空(页面将沿用模板现有内容);"
|
||||
"没有指示到的部分按源数据自由发挥,尽量把各分区填满。",
|
||||
"字段名必须与【目标结构】逐字一致(包括中文键名,原样复制),"
|
||||
"严禁自造、翻译或改写字段名;数组元素的字段集合必须与样例元素相同,"
|
||||
"源里没有的字段按类型留空即可,绝不能用自造的字段名顶替,也不要照抄任何示例值。",
|
||||
"另外请在输出的 JSON 里额外携带两个顶层字符串字段:「页面标题」(按主题拟一个"
|
||||
"页面大标题;若任务要求里明确给了标题,就用它)和「页面副标题」(一句话副标题)。",
|
||||
"",
|
||||
"【目标结构】(展示了完整字段与类型,其中数组只示意 1 个样例元素,实际请按源数据填充全部元素):",
|
||||
"【目标结构】(字段与类型的空占位骨架,数组只示意 1 个样例元素,实际按源数据填充全部元素):",
|
||||
"```json\n" + skeleton_json + "\n```",
|
||||
"",
|
||||
]
|
||||
if instructions and instructions.strip():
|
||||
parts.append("【任务要求】(用户对本次页面的要求/修改指示,优先级最高):")
|
||||
parts.append(instructions.strip())
|
||||
parts.append("")
|
||||
if embedded_json is not None:
|
||||
# 已从源数据里识别出结构化 JSON,单独标注帮助(尤其弱)模型对齐。
|
||||
parts.append("【从源数据中识别到的结构化数据】(优先以此为准):")
|
||||
@@ -1410,14 +1433,14 @@ class ScheduledTaskService:
|
||||
parts.append("【源数据】:")
|
||||
parts.append(source)
|
||||
parts.append("")
|
||||
parts.append("只输出一个 ```json 代码块,内含填充好的完整目标 JSON,不要任何解释文字。")
|
||||
parts.append("只输出一个 ```json 代码块,内含填充好的完整目标 JSON(含「页面标题」「页面副标题」),不要任何解释文字。")
|
||||
return "\n".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_prompt(base_prompt: str, errors: list[str], gaps: list[str]) -> str:
|
||||
def _build_retry_prompt(base_prompt: str, problems: list[str], gaps: list[str]) -> str:
|
||||
lines = [base_prompt, "\n\n上一轮转换结果未通过校验,请严格修正后重新输出**完整**的目标 JSON:"]
|
||||
if errors:
|
||||
lines.append("- 结构错误:" + ";".join(errors[:12]))
|
||||
if problems:
|
||||
lines.append("- " + ";".join(problems[:12]))
|
||||
if gaps:
|
||||
lines.append("- 字段缺失(请补齐):" + "、".join(gaps[:30]))
|
||||
return "\n".join(lines)
|
||||
@@ -1474,36 +1497,51 @@ class ScheduledTaskService:
|
||||
model_name: str | None,
|
||||
soul: str | None,
|
||||
run_id: str,
|
||||
instructions: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""把源数据(JSON/文本/含JSON文本)转换成模板的 DATA 结构,校验+回喂重试。
|
||||
|
||||
成功返回校验通过的对象;全部尝试失败抛 :class:`TemplateFillError`。
|
||||
通过(结构+字段对齐)返回校验过的对象;重试耗尽时**尽力而为**——只要解析
|
||||
出过 JSON 对象就返回最后一个(缺顶层键/字段错位的分支由随后的
|
||||
``merge_with_reference`` 回退模板默认数据,抽多少用多少);只有全程没解析
|
||||
出 JSON 才抛 :class:`TemplateFillError`(由调用方兜底为默认数据,不让
|
||||
run 失败)。
|
||||
"""
|
||||
skeleton = template_fill.schema_skeleton_json(template_id)
|
||||
# 源数据可能是纯 JSON / 纯文本 / 含 JSON 的文本:先尽力抽出内嵌 JSON 单独标注。
|
||||
embedded_json = template_fill.extract_embedded_json(source)
|
||||
base_prompt = self._build_conversion_prompt(skeleton, source, embedded_json)
|
||||
last_errors: list[str] = []
|
||||
base_prompt = self._build_conversion_prompt(skeleton, source, embedded_json, instructions=instructions)
|
||||
last_problems: list[str] = []
|
||||
prompt = base_prompt
|
||||
candidate: dict[str, Any] | None = None
|
||||
for attempt in range(_MAX_TEMPLATE_FILL_RETRIES + 1):
|
||||
raw = await self._invoke_conversion_model(model_name, soul, prompt)
|
||||
candidate = html_page_utils.extract_json_object(raw)
|
||||
if candidate is None:
|
||||
last_errors = ["模型未返回可解析的 JSON 对象"]
|
||||
parsed = html_page_utils.extract_json_object(raw)
|
||||
if parsed is None:
|
||||
last_problems = ["模型未返回可解析的 JSON 对象"]
|
||||
else:
|
||||
errors = template_fill.validate_data(template_id, candidate)
|
||||
if not errors:
|
||||
candidate = parsed # 记住最后一个可解析结果,供重试耗尽时尽力返回
|
||||
errors = [f"结构错误:{e}" for e in template_fill.validate_data(template_id, candidate)]
|
||||
mismatches = [f"字段错位:{m}" for m in template_fill.shape_mismatches(template_id, candidate)]
|
||||
if not errors and not mismatches:
|
||||
return candidate
|
||||
last_errors = errors
|
||||
last_problems = errors + mismatches
|
||||
gaps = template_fill.field_coverage_gaps(template_id, candidate)
|
||||
prompt = self._build_retry_prompt(base_prompt, errors, gaps)
|
||||
prompt = self._build_retry_prompt(base_prompt, last_problems, gaps)
|
||||
logger.info(
|
||||
"模板填充[run=%s]第 %d 次校验未通过:%s",
|
||||
run_id,
|
||||
attempt + 1,
|
||||
";".join(last_errors[:5]),
|
||||
";".join(last_problems[:5]),
|
||||
)
|
||||
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_errors[:8]))
|
||||
if candidate is not None:
|
||||
logger.warning(
|
||||
"模板填充[run=%s]重试耗尽仍未通过校验,尽力返回待兜底:%s",
|
||||
run_id,
|
||||
";".join(last_problems[:5]),
|
||||
)
|
||||
return candidate
|
||||
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_problems[:8]))
|
||||
|
||||
async def _finalize_template_html(
|
||||
self,
|
||||
@@ -1647,14 +1685,21 @@ class ScheduledTaskService:
|
||||
# HTML 数据来源:抽取(开关开 + 有数据)→ 转换填充;否则用模板默认数据(子页面齐全)。
|
||||
source = (data_text or "").strip()
|
||||
used_default = True
|
||||
page_title: str | None = None
|
||||
page_subtitle: str | None = None
|
||||
if do_extract and source:
|
||||
# 转换阶段同样带上任务所选智能体:SOUL 提供领域口径,未显式选模型时
|
||||
# 兜底用该智能体配置的模型(与模板填充模式一致)。
|
||||
soul, conv_model = await self._agent_soul_and_model(task, model_name)
|
||||
# 任务的描述/prompt 作为「任务要求」注入:用户对页面的修改指示(如
|
||||
# "标题改为…"、"只更新某部分")优先级最高,没指示就按检索内容自由发挥。
|
||||
instructions = "\n".join(part for part in (description, prompt) if part)
|
||||
try:
|
||||
converted = await self._convert_to_template_data(
|
||||
tpl_id, source, model_name=conv_model, soul=soul, run_id=run_id
|
||||
tpl_id, source, model_name=conv_model, soul=soul, run_id=run_id, instructions=instructions or None
|
||||
)
|
||||
# 页面标题/副标题是额外键,摘出来替换页头,不进 DATA。
|
||||
page_title, page_subtitle = template_fill.pop_page_titles(converted)
|
||||
# 子页面没抽到有效内容的,回退模板原有数据,避免子页面空白。
|
||||
data_obj = template_fill.merge_with_reference(tpl_id, converted)
|
||||
content = f"已检索数据并套用模板「{spec.name}」(空子页面回退原数据,含全部子页面)"
|
||||
@@ -1672,7 +1717,9 @@ class ScheduledTaskService:
|
||||
)
|
||||
|
||||
try:
|
||||
html = template_fill.fill_template(tpl_id, data_obj, update_date=local_time.date())
|
||||
html = template_fill.fill_template(
|
||||
tpl_id, data_obj, update_date=local_time.date(), title=page_title, subtitle=page_subtitle
|
||||
)
|
||||
except template_fill.TemplateError as exc:
|
||||
raise HtmlGenerationError(f"模板填充失败:{exc}") from exc
|
||||
return await self._finalize_template_html(
|
||||
@@ -1701,8 +1748,9 @@ class ScheduledTaskService:
|
||||
Pipeline: 取源 JSON(任务 prompt) → 用内置「模板数据转换器」agent 的 SOUL +
|
||||
模型把它映射成目标结构 → 结构校验(不过则回喂错误重试) → 填充模板 → 归档
|
||||
``index.html`` + 持久化 ``scheduled_html_pages`` → 返回 ``format: "html"``
|
||||
结果(复用现有 HTML 预览)。任一步彻底失败抛 :class:`TemplateFillError`,
|
||||
run 记为 failed。
|
||||
结果(复用现有 HTML 预览)。转换质量问题一律**不失败**:抽不到/抽错的分支由
|
||||
``merge_with_reference`` 回退模板原有数据,转换彻底失败则整页用模板默认数据;
|
||||
只有模板文件本身缺失/损坏才抛 :class:`TemplateFillError` 让 run 失败。
|
||||
"""
|
||||
template_id = _task_template_id(task)
|
||||
spec = template_fill.get_template(template_id) if template_id else None
|
||||
@@ -1726,6 +1774,8 @@ class ScheduledTaskService:
|
||||
used_default=True,
|
||||
)
|
||||
|
||||
page_title: str | None = None
|
||||
page_subtitle: str | None = None
|
||||
if not source:
|
||||
# 未提供源数据 → 用模板自带默认数据(不调模型、不校验)。
|
||||
data_obj = template_fill.reference_data(template_id)
|
||||
@@ -1740,18 +1790,33 @@ class ScheduledTaskService:
|
||||
model_name = str(tcfg.get("model_name") or "").strip() or None
|
||||
soul, model_name = await self._agent_soul_and_model(task, model_name)
|
||||
|
||||
converted = await self._convert_to_template_data(
|
||||
template_id, source, model_name=model_name, soul=soul, run_id=run_id
|
||||
)
|
||||
# 子页面没抽到有效内容的,回退模板原有数据,避免子页面空白。
|
||||
data_obj = template_fill.merge_with_reference(template_id, converted)
|
||||
content = f"已根据源数据填充模板「{spec.name}」生成页面"
|
||||
used_default = False
|
||||
try:
|
||||
converted = await self._convert_to_template_data(
|
||||
template_id, source, model_name=model_name, soul=soul, run_id=run_id
|
||||
)
|
||||
except TemplateFillError:
|
||||
# 转换彻底失败(全程没解析出 JSON)→ 也不让 run 失败:用模板默认
|
||||
# 数据出页,与 require_html 模板参考路径的兜底一致。
|
||||
logger.warning(
|
||||
"模板填充转换失败,回退默认数据 run=%s tpl=%s", run_id, template_id, exc_info=True
|
||||
)
|
||||
data_obj = template_fill.reference_data(template_id)
|
||||
content = f"数据转换未通过,已用模板「{spec.name}」的默认数据生成页面(含全部子页面)"
|
||||
used_default = True
|
||||
else:
|
||||
# 页面标题/副标题是额外键,摘出来替换页头,不进 DATA。
|
||||
page_title, page_subtitle = template_fill.pop_page_titles(converted)
|
||||
# 抽不到/抽错的子页面与字段回退模板原有数据,避免子页面空白或 undefined。
|
||||
data_obj = template_fill.merge_with_reference(template_id, converted)
|
||||
content = f"已根据源数据填充模板「{spec.name}」生成页面(抽取不到的部分沿用模板原有数据)"
|
||||
used_default = False
|
||||
|
||||
task_tz = ZoneInfo(task.get("timezone") or "UTC")
|
||||
local_time = scheduled_for.astimezone(task_tz)
|
||||
try:
|
||||
html = template_fill.fill_template(template_id, data_obj, update_date=local_time.date())
|
||||
html = template_fill.fill_template(
|
||||
template_id, data_obj, update_date=local_time.date(), title=page_title, subtitle=page_subtitle
|
||||
)
|
||||
except template_fill.TemplateError as exc:
|
||||
raise TemplateFillError(f"模板填充失败:{exc}") from exc
|
||||
return await self._finalize_template_html(
|
||||
|
||||
+202
-9
@@ -17,7 +17,8 @@ HTML 模板需要的 ``DATA`` 结构后,本模块负责把数据**填充**进模
|
||||
* **harness 层、零 app.* 依赖**:可被调度器(harness)与 Gateway 路由(app)共同 import。
|
||||
|
||||
公开入口:``list_templates`` / ``get_template`` / ``reference_data`` /
|
||||
``schema_skeleton_json`` / ``validate_data`` / ``field_coverage_gaps`` / ``fill_template``。
|
||||
``schema_skeleton_json`` / ``validate_data`` / ``field_coverage_gaps`` /
|
||||
``shape_mismatches`` / ``fill_template``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -406,25 +407,33 @@ def reference_data(template_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
def _skeleton(value: Any, *, list_sample: int = 1, depth: int = 0, max_depth: int = 6) -> Any:
|
||||
"""把一份具体数据压成「结构骨架」:保留全部字段与类型,每个数组只留前 N 个样例。
|
||||
"""把一份具体数据压成「空占位骨架」:保留全部字段名与类型,值一律抹空。
|
||||
|
||||
数组元素同构,留 1 个样例即可暴露该元素的完整字段集,既完整又紧凑——正好做大模型
|
||||
的「目标格式说明」。
|
||||
模板只是样式外壳,**示例值(模板自带的模拟内容)绝不能进入转换 prompt**——
|
||||
否则大模型会照抄示例内容、或被其领域(如台海防卫)限制住,不肯把用户主题的
|
||||
内容映射进来。叶子按类型给空占位:字符串→\"\"、数字→0、布尔→false、
|
||||
其它→null;数组元素同构,留 1 个抹空样例即可暴露该元素的完整字段集。
|
||||
"""
|
||||
if depth >= max_depth:
|
||||
return "…"
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
return {k: _skeleton(v, list_sample=list_sample, depth=depth + 1, max_depth=max_depth) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
return []
|
||||
return [_skeleton(v, list_sample=list_sample, depth=depth + 1, max_depth=max_depth) for v in value[:list_sample]]
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return False
|
||||
if isinstance(value, (int, float)):
|
||||
return 0
|
||||
if isinstance(value, str):
|
||||
return ""
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def schema_skeleton_json(template_id: str) -> str:
|
||||
"""目标格式的结构骨架(JSON 字符串),注入到大模型的转换 prompt 里。"""
|
||||
"""目标格式的空占位骨架(JSON 字符串),注入到大模型的转换 prompt 里。"""
|
||||
skeleton = _skeleton(reference_data(template_id))
|
||||
return json.dumps(skeleton, ensure_ascii=False, indent=1)
|
||||
|
||||
@@ -509,11 +518,134 @@ def _has_content(value: Any) -> bool:
|
||||
return True # 数字 / 布尔 / 其它标量
|
||||
|
||||
|
||||
# 元素字段中至少要有这个比例能在模板参考数据的元素字段里找到,否则视为「自造字段」。
|
||||
_LIST_FIELD_OVERLAP_MIN = 0.5
|
||||
# 元素字段至少要覆盖模板参考元素字段的这个比例:低于它意味着大部分展示字段缺失
|
||||
# (卡片大面积空白/undefined),不如整支回退模板原数据。
|
||||
_LIST_FIELD_RECALL_MIN = 0.4
|
||||
|
||||
|
||||
def _list_misaligned(got: Any, ref_val: Any) -> bool:
|
||||
"""非空列表与模板参考列表是否「无法有效渲染」。
|
||||
|
||||
顶层校验挡不住这种情况:转换结果键齐全、类型也对,但列表元素的**字段**不对,
|
||||
填进模板后 JS 读不到字段,页面满屏 ``undefined``。判定(满足其一即错位):
|
||||
|
||||
* 模板元素是对象而转换给了非对象(字符串/数字读不出属性);
|
||||
* 元素字段里能对上模板的比例过低(:data:`_LIST_FIELD_OVERLAP_MIN`,
|
||||
自造字段名占主导);
|
||||
* 元素字段对模板参考字段的**覆盖率**过低(:data:`_LIST_FIELD_RECALL_MIN`,
|
||||
只填了零星几个字段,展示字段大面积缺失)。
|
||||
|
||||
空列表不算错位(交由「无内容回退」处理)。
|
||||
"""
|
||||
if not isinstance(got, list) or not got:
|
||||
return False
|
||||
ref_fields = _element_fields(ref_val)
|
||||
if not ref_fields:
|
||||
return False # 模板元素本身没有对象字段,无从比对
|
||||
ref_dicts = [item for item in ref_val if isinstance(item, dict)]
|
||||
got_dicts = [item for item in got if isinstance(item, dict)]
|
||||
if ref_dicts and not got_dicts:
|
||||
return True # 模板元素是对象,转换给了字符串/数字 → 必然读不到字段
|
||||
if not got_dicts:
|
||||
return False
|
||||
got_fields: set[str] = set()
|
||||
for item in got_dicts:
|
||||
got_fields.update(item.keys())
|
||||
if not got_fields:
|
||||
return False
|
||||
overlap = got_fields & ref_fields
|
||||
if len(overlap) / len(got_fields) < _LIST_FIELD_OVERLAP_MIN:
|
||||
return True # 自造字段名占主导
|
||||
return len(overlap) / len(ref_fields) < _LIST_FIELD_RECALL_MIN # 覆盖率过低
|
||||
|
||||
|
||||
def _mismatch_reason(got: Any, ref_val: Any) -> str:
|
||||
"""错位原因的人话描述(回喂重试用),与 :func:`_list_misaligned` 同判定。"""
|
||||
ref_fields = _element_fields(ref_val)
|
||||
got_fields: set[str] = set()
|
||||
for item in got or []:
|
||||
if isinstance(item, dict):
|
||||
got_fields.update(item.keys())
|
||||
overlap = got_fields & ref_fields
|
||||
if got_fields and len(overlap) / len(got_fields) < _LIST_FIELD_OVERLAP_MIN:
|
||||
return "的元素字段名与模板不一致(疑似自造字段名,必须原样沿用目标结构的字段名)"
|
||||
return "的元素字段覆盖过低(样例元素的字段大部分缺失;缺数据的字段按类型留空补齐,不要省略字段)"
|
||||
|
||||
|
||||
def shape_mismatches(template_id: str, obj: Any, *, max_items: int = 20) -> list[str]:
|
||||
"""硬反馈:列出非空列表分支里「无法有效渲染」的项(供回喂重试)。
|
||||
|
||||
与 :func:`field_coverage_gaps`(软反馈,字段没填全)不同,本函数抓的是
|
||||
**填了也渲染不出来**的情况:列表有内容但元素字段自造(对不上模板)或覆盖
|
||||
过低(展示字段大面积缺失) → 渲染必然大面积 undefined,必须回喂重试。
|
||||
"""
|
||||
out: list[str] = []
|
||||
if not isinstance(obj, dict):
|
||||
return out
|
||||
ref = reference_data(template_id)
|
||||
for key, ref_val in ref.items():
|
||||
got = obj.get(key)
|
||||
if isinstance(ref_val, list) and _list_misaligned(got, ref_val):
|
||||
out.append(f"「{key}」{_mismatch_reason(got, ref_val)}")
|
||||
if len(out) >= max_items:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _ref_element_types(ref_val: list) -> dict[str, Any]:
|
||||
"""参考列表各字段的类型样本(取第一个携带该字段的元素)。"""
|
||||
types: dict[str, Any] = {}
|
||||
for item in ref_val:
|
||||
if isinstance(item, dict):
|
||||
for key, value in item.items():
|
||||
types.setdefault(key, value)
|
||||
return types
|
||||
|
||||
|
||||
def _blank_like(sample: Any) -> Any:
|
||||
"""按参考类型给出「空值」:字符串→空串、列表→空数组、对象→空对象、其它→null。"""
|
||||
if isinstance(sample, str):
|
||||
return ""
|
||||
if isinstance(sample, list):
|
||||
return []
|
||||
if isinstance(sample, dict):
|
||||
return {}
|
||||
return None
|
||||
|
||||
|
||||
def _fill_missing_fields(got: list, ref_val: list) -> list:
|
||||
"""给保留的转换列表逐元素补齐模板元素字段(缺失的按参考类型给空值)。
|
||||
|
||||
模板 JS 常裸读 ``${item.字段}``(不经过空值兜底函数),字段缺失会直接把
|
||||
``undefined`` 打到页面上;补齐成空值后渲染为空白,不再出现字面 undefined。
|
||||
只补缺、不改已有值,多出来的自造键原样保留(JS 不读,无害)。
|
||||
"""
|
||||
types = _ref_element_types(ref_val)
|
||||
if not types:
|
||||
return got
|
||||
out: list[Any] = []
|
||||
for item in got:
|
||||
if isinstance(item, dict):
|
||||
filled = dict(item)
|
||||
for key, sample in types.items():
|
||||
if key not in filled:
|
||||
filled[key] = _blank_like(sample)
|
||||
out.append(filled)
|
||||
else:
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def _merge_fill(got: Any, ref: Any) -> Any:
|
||||
"""用 ``got`` 覆盖,但 ``got`` 里「没有有效内容」的部分回退 ``ref``。
|
||||
|
||||
字典逐键递归(空分支回退);列表/标量有内容就整体保留 ``got``(列表不逐元素合并,
|
||||
避免与原模板元素错配)。
|
||||
字典逐键递归(空分支回退);列表有内容就整体保留 ``got``(不逐元素合并,避免与
|
||||
原模板元素错配)——**例外**:列表元素无法有效渲染(:func:`_list_misaligned`,
|
||||
自造字段名占主导/覆盖率过低/元素类型不符)时整支回退 ``ref``,否则填进模板
|
||||
JS 读不到字段,页面满屏 undefined。保留的列表会逐元素补齐缺失字段的空值
|
||||
(:func:`_fill_missing_fields`),防止局部缺字段裸读出 undefined。
|
||||
"""
|
||||
if not _has_content(got):
|
||||
return ref
|
||||
@@ -522,6 +654,10 @@ def _merge_fill(got: Any, ref: Any) -> Any:
|
||||
for key, ref_val in ref.items():
|
||||
out[key] = _merge_fill(got.get(key), ref_val)
|
||||
return out
|
||||
if isinstance(ref, list) and isinstance(got, list):
|
||||
if _list_misaligned(got, ref):
|
||||
return ref
|
||||
return _fill_missing_fields(got, ref)
|
||||
return got
|
||||
|
||||
|
||||
@@ -541,21 +677,78 @@ def _dump_const(var: str, obj: Any) -> str:
|
||||
return f"const {var}={body};"
|
||||
|
||||
|
||||
def _escape_html(text: str) -> str:
|
||||
return (
|
||||
text.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
)
|
||||
|
||||
|
||||
# 转换结果里约定的两个额外顶层键:页面大标题/副标题(填充时替换页头,不进 DATA)。
|
||||
PAGE_TITLE_KEY = "页面标题"
|
||||
PAGE_SUBTITLE_KEY = "页面副标题"
|
||||
|
||||
|
||||
def pop_page_titles(obj: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
"""取出并移除转换结果里的「页面标题/页面副标题」额外键。
|
||||
|
||||
模型按转换 prompt 的约定额外输出这两个顶层字符串键;它们不是模板 DATA 的
|
||||
字段(校验本就忽略额外键),填充前摘出来用于替换页头,避免漏进 ``const DATA``。
|
||||
"""
|
||||
def _clean(value: Any) -> str | None:
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
title = _clean(obj.pop(PAGE_TITLE_KEY, None))
|
||||
subtitle = _clean(obj.pop(PAGE_SUBTITLE_KEY, None))
|
||||
return title, subtitle
|
||||
|
||||
|
||||
def fill_template(
|
||||
template_id: str,
|
||||
data_obj: Any,
|
||||
*,
|
||||
manifest_overrides: dict[str, Any] | None = None,
|
||||
update_date: date | None = None,
|
||||
title: str | None = None,
|
||||
subtitle: str | None = None,
|
||||
) -> str:
|
||||
"""把 ``data_obj`` 填进模板的 ``const DATA=…`` 行,产出完整 HTML 文档。
|
||||
|
||||
可选地更新 MANIFEST 的「系统.更新时间」(``update_date``,默认不动) 以及任意
|
||||
``manifest_overrides``(深合并到解析后的 MANIFEST,如改系统名称)。
|
||||
|
||||
``title``/``subtitle`` 来自转换结果里的「页面标题/页面副标题」:模板页头
|
||||
(<title>、第一个 <h1>、其后紧跟的副标题 <p>)是写死的静态 HTML,不随 DATA
|
||||
变化——给出新标题时替换这三处,并把 MANIFEST 的 系统.名称 一并同步;
|
||||
不给则保持模板原文。
|
||||
"""
|
||||
spec = _require(template_id)
|
||||
text = read_template_text(template_id)
|
||||
|
||||
if title:
|
||||
overrides = dict(manifest_overrides or {})
|
||||
system = overrides.get("系统")
|
||||
system = dict(system) if isinstance(system, dict) else {}
|
||||
system["名称"] = title
|
||||
overrides["系统"] = system
|
||||
manifest_overrides = overrides
|
||||
escaped = _escape_html(title)
|
||||
text = re.sub(r"<title>.*?</title>", lambda _m: f"<title>{escaped}</title>", text, count=1, flags=re.DOTALL)
|
||||
text = re.sub(r"<h1>.*?</h1>", lambda _m: f"<h1>{escaped}</h1>", text, count=1, flags=re.DOTALL)
|
||||
if subtitle:
|
||||
escaped_sub = _escape_html(subtitle)
|
||||
text = re.sub(
|
||||
r"(<h1>.*?</h1>\s*<p>).*?(</p>)",
|
||||
lambda m: f"{m.group(1)}{escaped_sub}{m.group(2)}",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
data_re = _const_line_re(spec.data_var)
|
||||
if data_re.search(text) is None:
|
||||
raise TemplateError(f"模板 {template_id} 中未找到 const {spec.data_var}= 行")
|
||||
|
||||
Reference in new issue
Block a user