feat(deep-research): 强化任务链路可靠性并接入 agentfx 报告助手种子

功能说明:\n- 重构 Deep Research 调度与执行链,补齐 dispatcher、executor、live hub 与依赖注入入口,提升任务下发、执行、回调的稳定性\n- 增强运行时持久化与恢复能力,接入业务映射、任务来源与任务事件日志模型,覆盖持久化与执行可靠性关键路径测试\n- 新增 agentfx 报告助手 seed 与任务报告导入脚本配置,支持离线环境快速初始化与任务流模板回填\n- 完善 deep-research 与 thread_runs 路由参数编排,兼容不同业务映射与运行入口的前后端联动\n- 补充前端 agentfx deep-link 解析,保证 deep-research 场景下的登录跳转与任务上下文一致
This commit is contained in:
董宏凯 committed 2026-09-04 23:18:52 +08:00
1 parent 438772f0b7
commit 5c75e1af16
36 files changed
+3163 -312

No files matched your search

@@ -16,8 +16,10 @@ from __future__ import annotations
import json
import logging
from collections.abc import Awaitable, Callable
from contextlib import suppress
from datetime import date, datetime
from enum import Enum
from time import monotonic
from typing import TYPE_CHECKING, Any
from deerflow.agents.deep_research.events import (
@@ -42,6 +44,20 @@ def _json_default(value: Any) -> Any:
return str(value)
# Live-only frame types that have no durable counterpart at all. Report and
# summary text are already checkpointed (``report_chunk`` / ``summary_completed``),
# so only the reasoning channel needs a cross-worker mirror.
_MIRRORED_LIVE_TYPES = frozenset({"report_thinking", "plan_reasoning", "summary_thinking"})
# Coalescing bounds for that mirror: one bounded row per ~600 characters or
# ~1.5 seconds, never one row per provider token.
_MIRROR_MIN_CHARS = 600
_MIRROR_MAX_DELAY_S = 1.5
# Floor for the time-based flush: a slowly trickling reasoning stream must not
# degrade into one durable row per token. Anything shorter is carried until the
# next durable milestone flushes it.
_MIRROR_MIN_FLUSH_CHARS = 40
def _json_safe_payload(payload: dict[str, Any]) -> dict[str, Any]:
"""Normalize event payloads before they reach any event repository.
@@ -66,11 +82,18 @@ class PersistingEventSink:
session_id: str,
job_id: str,
on_event: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
live_subscriber_probe: Callable[[str], bool] | None = None,
) -> None:
self._repo = repo
self._session_id = session_id
self._job_id = job_id
self._on_event = on_event
# Reports whether this worker holds an SSE connection for the job.
# Absent probe → assume the live path reaches the viewer (single-worker
# deployments and tests keep today's zero-extra-write behaviour).
self._live_subscriber_probe = live_subscriber_probe
self._mirror_buffers: dict[str, dict[str, Any]] = {}
self._mirroring = False
self.diagnostics: list[dict[str, Any]] = []
# A durable event row normally owns the sequence. When the event
# database is temporarily unavailable, use distinct negative sequence
@@ -87,6 +110,10 @@ class PersistingEventSink:
) -> DeepResearchEvent:
event_payload = _json_safe_payload(payload or {})
persistence_error: Exception | None = None
if not self._mirroring:
# Order any coalesced reasoning tail before the next milestone
# instead of leaving it buffered until the job ends.
await self._flush_live_mirrors()
try:
row = await self._repo.append(
session_id=self._session_id,
@@ -211,6 +238,87 @@ class PersistingEventSink:
type,
exc_info=True,
)
with suppress(Exception):
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
# ── cross-worker mirror for live-only reasoning frames ──────────────────
async def _mirror_live_frame(
self,
type: str, # noqa: A002
*,
phase: ResearchPhase,
payload: dict[str, Any],
) -> None:
"""Buffer a live-only reasoning delta when no local viewer can see it.
The live hub is in-process. A job runs on the worker that won its
durable lease, while the SSE connection is served by whichever worker
accepted it, so on a multi-worker deployment the model's thinking is
usually published into a hub with no subscribers and is then lost —
there is no durable row to replay it from. When that is the case, the
reasoning is coalesced here and persisted as a bounded durable event
so the polling stream on the other worker still renders it.
"""
if type not in _MIRRORED_LIVE_TYPES or self._live_subscriber_probe is None:
return
delta = payload.get("delta")
if not isinstance(delta, str) or not delta:
return
if self._live_subscriber_probe(self._job_id):
# A viewer is attached here and has already received every frame
# live. Drop the buffer rather than flushing it, so that viewer
# does not also receive the same text as a durable replay.
self._mirror_buffers.pop(self._mirror_key(type, payload), None)
return
key = self._mirror_key(type, payload)
buffered = self._mirror_buffers.get(key)
if buffered is None:
buffered = {
"type": type,
"phase": phase,
"target": payload.get("target"),
"reportVersion": payload.get("reportVersion"),
"text": "",
"since": monotonic(),
}
self._mirror_buffers[key] = buffered
buffered["text"] += delta
buffered["phase"] = phase
buffered_chars = len(buffered["text"])
aged_out = (
buffered_chars >= _MIRROR_MIN_FLUSH_CHARS
and monotonic() - float(buffered["since"]) >= _MIRROR_MAX_DELAY_S
)
if buffered_chars >= _MIRROR_MIN_CHARS or aged_out:
await self._emit_mirror(key)
@staticmethod
def _mirror_key(type: str, payload: dict[str, Any]) -> str: # noqa: A002
return f"{type}::{payload.get('target') or 'report'}"
async def _flush_live_mirrors(self) -> None:
for key in tuple(self._mirror_buffers):
with suppress(Exception):
await self._emit_mirror(key)
async def _emit_mirror(self, key: str) -> None:
buffered = self._mirror_buffers.pop(key, None)
if buffered is None or not buffered["text"]:
return
payload: dict[str, Any] = {
"delta": buffered["text"],
"liveMirror": True,
}
if buffered["target"] is not None:
payload["target"] = buffered["target"]
if buffered["reportVersion"] is not None:
payload["reportVersion"] = buffered["reportVersion"]
self._mirroring = True
try:
await self.emit(buffered["type"], phase=buffered["phase"], payload=payload)
finally:
self._mirroring = False
class NullEventSink:
@@ -33,7 +33,7 @@ from deerflow.agents.deep_research.cancellation import CancellationToken
from deerflow.agents.deep_research.config import DeepResearchConfig, ModelRole
from deerflow.agents.deep_research.events import EventSink
from deerflow.agents.deep_research.types import CompletionResult
from deerflow.utils.stream_text import InlineThinkTagFilter
from deerflow.utils.stream_text import InlineThinkTagFilter, strip_inline_think_tags
if TYPE_CHECKING:
from langchain.chat_models import BaseChatModel
@@ -388,31 +388,57 @@ class DeerFlowCompletionBackend:
# Custom ``report_delta`` events remain the only live preview channel.
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
attempts = 2 if operation.startswith("write_report") else 1
last_error: Exception | None = None
try:
async for chunk in model.astream(
to_langchain_messages(messages),
config=isolated_config,
**kwargs,
):
if self._cancel is not None:
self._cancel.raise_if_cancelled()
reasoning = _reasoning_delta(chunk)
if reasoning and on_reasoning is not None:
await on_reasoning(reasoning)
# A few OpenAI-compatible models put their reasoning in the
# ordinary content channel instead of ``reasoning_content``.
# Split it from Markdown even when tags span tokens: callers may
# display a live thought trace, but it never joins ``text``.
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
if inline_reasoning and on_reasoning is not None:
await on_reasoning(inline_reasoning)
if delta:
text_parts.append(delta)
if on_delta is not None:
await on_delta(delta)
chunk_usage = _extract_usage(chunk)
if any(chunk_usage.values()):
usage = chunk_usage
for attempt in range(attempts):
text_parts.clear()
inline_think_filter = InlineThinkTagFilter()
try:
async for chunk in model.astream(
to_langchain_messages(messages),
config=isolated_config,
**kwargs,
):
if self._cancel is not None:
self._cancel.raise_if_cancelled()
reasoning = _reasoning_delta(chunk)
if reasoning and on_reasoning is not None:
await on_reasoning(reasoning)
# A few OpenAI-compatible models put their reasoning in the
# ordinary content channel instead of ``reasoning_content``.
# Split it from Markdown even when tags span tokens: callers may
# display a live thought trace, but it never joins ``text``.
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
if inline_reasoning and on_reasoning is not None:
await on_reasoning(inline_reasoning)
if delta:
text_parts.append(delta)
if on_delta is not None:
await on_delta(delta)
chunk_usage = _extract_usage(chunk)
if any(chunk_usage.values()):
usage = chunk_usage
last_error = None
break
except Exception as exc: # noqa: BLE001 - report writing waits like ordinary chat
from deerflow.agents.deep_research.cancellation import ResearchCancelled
if isinstance(exc, (ResearchCancelled, asyncio.CancelledError)):
raise
last_error = exc
# Only replay a clean connection; a half-written stream is
# reset by the runner, not by appending a second attempt.
if text_parts or attempt + 1 >= attempts:
raise
logger.warning(
"deep_research llm: %s stream interrupted before the first token (%s); retrying once",
operation,
type(exc).__name__,
)
await asyncio.sleep(1.5)
if last_error is not None:
raise last_error
finally:
reset_parent_config()
@@ -470,7 +496,10 @@ class DeerFlowCompletionBackend:
if self._cancel is not None:
self._cancel.raise_if_cancelled()
text = _content_to_text(response)
# ``ainvoke`` returns raw content, including the inline ``<think>``
# block some OpenAI-compatible deployments write there. The streamed
# path filters it out, so this retry path must not reintroduce it.
text = strip_inline_think_tags(_content_to_text(response))
usage = _extract_usage(response)
await self._record_usage(model_name, operation, usage)
return CompletionResult(text=text, model=model_name, usage=usage)
@@ -80,6 +80,13 @@ EventType = Literal[
"deep_branch_completed",
"report_reset",
"report_chunk",
# Model reasoning normally travels as a live-only frame. It becomes a
# durable event when no SSE connection for the job is attached to the
# worker running it, because the in-process live hub cannot reach a
# viewer served by a different worker (see PersistingEventSink).
"report_thinking",
"plan_reasoning",
"summary_thinking",
"report_completed",
"summary_completed",
"collection_completed",
@@ -22,6 +22,7 @@ vendored prompt families in a later commit without changing the runner shape.
from __future__ import annotations
import asyncio
import json
import logging
import re
@@ -46,6 +47,23 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Delta size used to replay a non-streamed retry report. Matches the durable
# chunk size so the retry produces the same event volume as a streamed write.
_RETRY_REPLAY_CHUNK = 480
_WRITE_REPORT_TIMEOUT_SECONDS = 180.0
async def _await_writer(coro):
"""Fail a hung production stream before the job lease is reclaimed."""
try:
return await asyncio.wait_for(coro, timeout=_WRITE_REPORT_TIMEOUT_SECONDS)
except TimeoutError as exc:
raise CompletionError(
f"Report writer exceeded {_WRITE_REPORT_TIMEOUT_SECONDS:g} seconds",
code="write_report_timeout",
retryable=True,
) from exc
# ── structured-output schemas ───────────────────────────────────────────────
@@ -508,11 +526,12 @@ class BasicRunner:
*,
collection_outcomes: list[dict[str, Any]] | None = None,
) -> DeepResearchResult:
"""Write a complete fallback report when collection returned no rows.
"""Write a complete report when collection returned no rows.
Research collection is best-effort. A collector timeout must not be
persisted as a "cannot generate" pseudo-report, regardless of which
runner mode is active.
Stream the same model writer as a normal report so the sandbox stays
empty through thinking and fills token-by-token. A refusal or transport
failure still falls through ``_generate_report`` to the deterministic
template; do not dump that template before the model has written.
"""
events = adapters.events
await events.emit(
@@ -520,7 +539,7 @@ class BasicRunner:
phase="curating",
payload={
"code": "no_materials",
"message": "未收集到外部研究材料,已改用领域知识继续撰写完整报告",
"message": "未收集到外部研究材料,将按公开领域知识撰写完整报告",
"recoverable": True,
"diagnostic": {
"stage": "collecting",
@@ -529,30 +548,55 @@ class BasicRunner:
},
},
)
await events.emit(
"report_reset",
phase="writing",
payload={"reportVersion": 1, "reason": "no_materials_write"},
)
await events.emit(
"phase_changed",
phase="writing",
payload={"from": "curating", "to": "writing"},
)
context_text = (
"本次没有可引用的外部研究材料。请只依据公开领域常识,就课题写成一份"
"完整、审慎的分析报告。不要拒绝生成,也不要把「没有资料」写成全文主线;"
"在局限性中说明证据边界即可。"
)
report_md = await self._generate_report(
adapters.llm,
request,
_KNOWLEDGE_FILL_EMPTY_CONTEXT,
context_text,
events,
cancellation,
allow_knowledge_fill=True,
source_count=0,
)
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
await adapters.artifacts.write_text("report.md", report_md)
await adapters.artifacts.write_text("sources.json", "[]")
await adapters.artifacts.write_text("research-plan.json", json.dumps(plan, ensure_ascii=False, indent=2))
await events.emit("report_completed", phase="writing", payload={"reportVersion": 1, "sourceCount": 0})
await events.emit(
"artifact_created",
phase="exporting",
payload={"format": "markdown", "name": "report.md"},
)
image_paths: dict[str, str] = {}
try:
report_md, image_paths = await generate_report_images(request, adapters, report_md, cancellation)
except ResearchCancelled:
raise
except Exception: # noqa: BLE001 - illustrations must not discard the report
logger.warning("no-materials image generation failed; retaining the text report", exc_info=True)
for name, content in (
("report.md", report_md),
("sources.json", "[]"),
("research-plan.json", json.dumps(plan, ensure_ascii=False, indent=2)),
):
try:
await adapters.artifacts.write_text(name, content)
except Exception: # noqa: BLE001 - session persistence still receives the report
logger.warning("no-materials artifact write failed for %s", name, exc_info=True)
try:
await events.emit("report_completed", phase="writing", payload={"reportVersion": 1, "sourceCount": 0})
await events.emit(
"artifact_created",
phase="exporting",
payload={"format": "markdown", "name": "report.md"},
)
except Exception: # noqa: BLE001 - completion events must not drop the report
logger.warning("no-materials completion events failed; retaining the report", exc_info=True)
return DeepResearchResult(
report_markdown=report_md,
source_ids=[],
@@ -619,30 +663,110 @@ class BasicRunner:
if request.config.structure_mode == "fixed" and (outline or "").strip():
system += "撰写时必须原样沿用用户大纲中的章节标题与层级,不得改写成通用报告套式。"
completion_messages = [
{"role": "system", "content": system},
{"role": "user", "content": prompt},
]
async def write_once(*, report_version: int) -> str:
emitter = ReportChunkEmitter(events, report_version=report_version)
if filled_preamble:
await emitter.add(filled_preamble + "\n\n")
completion_messages = [
{"role": "system", "content": system},
{"role": "user", "content": prompt},
]
result = await llm.stream_complete(
model_role="smart",
messages=completion_messages,
operation="write_report",
on_delta=emitter.add,
on_reasoning=emitter.add_thinking,
result = await _await_writer(
llm.stream_complete(
model_role="smart",
messages=completion_messages,
operation="write_report",
on_delta=emitter.add,
on_reasoning=emitter.add_thinking,
)
)
await emitter.flush()
return ensure_outline_preamble(result.text, outline, request.query)
async def retry_report(
*,
report_version: int,
reason: str,
code: str,
message: str,
error_type: str,
error_detail: str,
) -> str:
"""Spend one real second model call before any deterministic report.
A broken ``astream`` says nothing about the model's ability to
answer: deployments that stall past the inter-chunk deadline (idle
proxy, long reasoning pause) still return a complete response.
Dropping straight to the template silently downgrades a recoverable
transport failure into a material dump that looks like a success.
"""
await events.emit(
"warning",
phase="writing",
payload={
"code": code,
"message": message,
"recoverable": True,
"errorType": error_type,
"diagnostic": {
"stage": "writing",
"outcome": reason,
"errorType": error_type,
"detail": error_detail[:1200] or error_type,
},
},
)
await events.emit(
"report_reset",
phase="writing",
payload={"reportVersion": report_version, "reason": reason},
)
emitter = ReportChunkEmitter(events, report_version=report_version)
# Prefer the non-streaming transport when the adapter offers it: a
# stream that already died on an inter-chunk deadline usually dies
# the same way again, while ``ainvoke`` waits for the whole answer.
complete_prose = getattr(llm, "complete_prose", None)
if callable(complete_prose):
result = await _await_writer(
complete_prose(
model_role="smart",
messages=completion_messages,
operation="write_report_retry",
)
)
report = ensure_outline_preamble(result.text, outline, request.query)
# Replay as bounded deltas so the live sandbox and the durable
# chunk log see the same shape as a streamed write.
for index in range(0, len(report), _RETRY_REPLAY_CHUNK):
await emitter.add(report[index : index + _RETRY_REPLAY_CHUNK])
else:
result = await _await_writer(
llm.stream_complete(
model_role="smart",
messages=completion_messages,
operation="write_report_retry",
on_delta=emitter.add,
on_reasoning=emitter.add_thinking,
)
)
report = ensure_outline_preamble(result.text, outline, request.query)
await emitter.flush()
if not report.strip():
raise CompletionError(
"Report retry returned empty prose",
code="empty_report_retry",
retryable=False,
)
return report
async def complete_with_fallback(
*,
reason: str,
message: str,
error_type: str | None = None,
error_detail: str | None = None,
report_version: int = 2,
) -> str:
"""Replace an invalid streamed draft with a usable deterministic report.
@@ -668,39 +792,81 @@ class BasicRunner:
await events.emit(
"report_reset",
phase="writing",
payload={"reportVersion": 2, "reason": reason},
payload={"reportVersion": report_version, "reason": reason},
)
fallback = ensure_outline_preamble(
_build_report_fallback(request.query, context_text, source_count or 0),
outline,
request.query,
)
emitter = ReportChunkEmitter(events, report_version=2)
emitter = ReportChunkEmitter(events, report_version=report_version)
await emitter.add(fallback)
await emitter.flush()
return fallback
async def refusal_fallback() -> str:
return await complete_with_fallback(
reason="refusal_after_retry",
message="模型两次返回拒绝文本,已根据可用材料生成保底报告",
error_type="InvalidReportOutput",
error_detail=report,
report_version=3,
)
try:
report = await write_once(report_version=1)
except Exception as exc: # noqa: BLE001 - preserve materials without a second model call
if not _is_retryable_report_write_error(exc):
except ResearchCancelled:
raise
except Exception as exc: # noqa: BLE001 - writing must finish even on a transport blip
try:
report = await retry_report(
report_version=2,
reason="transient_write_error",
code="report_stream_retry",
message="报告写入连接中断,正在重试生成(已收集素材保持不变)",
error_type=type(exc).__name__,
error_detail=str(exc),
)
except ResearchCancelled:
raise
return await complete_with_fallback(
reason="report_stream_error",
message="报告写入连接中断,已根据可用材料生成保底报告",
error_type=type(exc).__name__,
error_detail=str(exc),
)
except Exception as retry_exc: # noqa: BLE001 - now use the deterministic report
return await complete_with_fallback(
reason="transient_write_error_after_retry",
message="报告写入连接中断且重试未成功,已根据可用材料生成保底报告",
error_type=type(retry_exc).__name__,
error_detail=(
f"stream={type(exc).__name__}: {str(exc)[:600]}; "
f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}"
),
report_version=3,
)
return report if not _is_refusal_report(report) else await refusal_fallback()
if not _is_refusal_report(report):
return report
return await complete_with_fallback(
reason="refusal_output",
message="模型返回了拒绝文本,已根据可用材料生成保底报告",
error_type="InvalidReportOutput",
error_detail=report,
)
# A refusal is a content failure, not a transport failure: one clean
# rewrite recovers it far more often than a material dump does.
try:
report = await retry_report(
report_version=2,
reason="invalid_refusal_output",
code="report_refusal_retry",
message="模型返回了拒绝文本,正在重试生成报告",
error_type="InvalidReportOutput",
error_detail=report,
)
except ResearchCancelled:
raise
except Exception as retry_exc: # noqa: BLE001 - keep the confirmed materials
return await complete_with_fallback(
reason="refusal_output",
message="模型返回了拒绝文本且重试失败,已根据可用材料生成保底报告",
error_type=type(retry_exc).__name__,
error_detail=f"retry={type(retry_exc).__name__}: {str(retry_exc)[:600]}",
report_version=3,
)
return report if not _is_refusal_report(report) else await refusal_fallback()
# ── helpers ─────────────────────────────────────────────────────────────────
@@ -998,7 +1164,29 @@ def _is_retryable_report_write_error(exc: Exception) -> bool:
if isinstance(exc, (TimeoutError, ConnectionError, OSError)):
return True
name = type(exc).__name__.lower()
return any(token in name for token in ("timeout", "connection", "network", "streamchunk"))
text = f"{name} {exc}".lower()
return any(
token in text
for token in (
"timeout",
"connection",
"network",
"streamchunk",
"readtimeout",
"connecterror",
"remoteprotocol",
"chunkedencoding",
"apiconnection",
"apitimeout",
"ratelimit",
"unavailable",
"badgateway",
"gateway",
"502",
"503",
"504",
)
)
def _error_diagnostic(stage: str, exc: Exception, *, query: str | None = None) -> dict[str, str]:
@@ -1039,23 +1227,27 @@ def _build_report_fallback(query: str, context_text: str, source_count: int) ->
## 摘要
本报告围绕“{query}”形成基础研究框架。本次外部检索未返回可引用来源,因此以下内容仅用于明确研究范围、分析维度与后续核验路径。
本报告围绕“{query}”给出可继续推进的研究框架。外部检索通道本次没有返回可引用条目,正文依据公开领域常识整理分析维度,供后续核验使用。
## 研究范围与方法
围绕课题梳理核心对象、时间范围、影响因素和可验证指标;后续应补充权威公开数据、政策文件、原始访谈或业务台账进行交叉验证。
围绕课题梳理核心对象、时间范围、影响因素和可验证指标。后续应补充权威公开数据、政策文件、原始访谈或业务台账,再对关键判断做交叉验证。
## 基础分析框架
可从现状与背景、关键驱动因素、风险与不确定性、利益相关方影响及可执行建议五个方面展开研判。所有具体事实、数值和结论均应在补充来源后确认。
可从现状与背景、关键驱动因素、风险与不确定性、利益相关方影响及可执行建议五个方面展开研判。凡涉及具体事实、数值和因果判断,均应在补充来源后确认。
## 后续核验建议
优先补齐时间线、主体关系、公开统计口径和相互独立的原始记录。每一条新增材料都应对回本框架中的对应问题,避免只堆叠摘要。
## 局限性
本次没有外部来源可供引用,不能将本报告中的研究框架视为已完成的事实核验。
本次成稿未附外部引用条目。文中框架用于界定问题,不等于已经完成事实核验。
## 结论
建议以本框架为基础继续补充可追溯材料,并在材料到位后更新各项实证结论。
建议以本框架为基础继续补充可追溯材料,并在材料到位后更新各项实证结论。即使用户侧素材通道暂时不可用,研究报告也应先完成这一版可阅读成稿。
"""
bounded_context = (context_text or "").strip()
@@ -1,10 +1,10 @@
"""ORM model for the global business → agent / chain mapping (业务链条管理).
A "business mapping" row binds a fixed business code (3Q / 6BF / 7BF / 8BF) to an
optional 智能体 (``agent_id``) and an optional 业务链条 (``chain_id`` → a
``roundtable_chains`` row, which must be **public** so every account can resolve
it). Used to auto-pick the business chain when a task page launches the
multi-agent roundtable.
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
there is no ``user_id`` column; all accounts see and use the same configuration.
@@ -1,7 +1,7 @@
"""SQLAlchemy-backed global business → agent / chain mapping store.
Global / shared (no ``user_id`` scoping): every account reads and writes the same
rows. The four business rows are seeded once on startup (``ensure_seed_defaults``)
rows. Fixed business rows are seeded once on startup (``ensure_seed_defaults``)
with empty agent/chain; users fill those in from the management page via
``update_mapping``.
"""
@@ -21,12 +21,22 @@ _UNSET = object()
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
{"business_code": "7BF", "label": "XDFX"},
{"business_code": "8BF", "label": "8BF"},
{"business_code": "AFX", "label": "AGENTFX"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
{"business_code": "6-4", "label": "6-4"},
{"business_code": "7-1", "label": "7-1"},
{"business_code": "7-2", "label": "7-2"},
{"business_code": "7-3", "label": "7-3"},
{"business_code": "7-4", "label": "7-4"},
]
@@ -94,14 +104,24 @@ class BusinessMappingRepository:
async def ensure_seed_defaults(self) -> int:
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
Never overwrites an existing row (respects user-configured agent/chain).
Never overwrites an existing row's agent/chain (respects user config).
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
the intended table order next to older seeded rows.
"""
inserted = 0
dirty = False
now = datetime.now(UTC)
async with self._sf() as session:
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
existing = await session.get(BusinessMappingRow, seed["business_code"])
if existing is not None:
if existing.sort_order != idx:
existing.sort_order = idx
dirty = True
seed_label = str(seed.get("label") or "")
if seed_label and not (existing.label or "").strip():
existing.label = seed_label
dirty = True
continue
session.add(
BusinessMappingRow(
@@ -117,6 +137,7 @@ class BusinessMappingRepository:
)
)
inserted += 1
if inserted:
dirty = True
if dirty:
await session.commit()
return inserted
@@ -137,6 +137,38 @@ class DeepResearchJobRepository:
# Should not happen in practice.
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
async def replace_input_snapshot(self, job_id: str, input_snapshot: dict[str, Any]) -> dict[str, Any] | None:
"""Overwrite the frozen start payload on a still-queued job.
Idempotent retries used to keep the first snapshot forever. A first
write that raced ahead of the browser evidence (or a reused empty
job) then ran without ``collector_materials`` even though the later
request carried them.
"""
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow)
.where(
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.in_(("queued", "running")),
)
.values(
input_snapshot=_dumps(input_snapshot),
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
await session.rollback()
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (
@@ -198,12 +230,13 @@ class DeepResearchJobRepository:
stmt = (
select(DeepResearchJobRow.id)
.where(
DeepResearchJobRow.cancel_requested.is_(False),
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now),
)
),
)
.order_by(DeepResearchJobRow.updated_at.asc())
.limit(limit)
@@ -227,6 +260,7 @@ class DeepResearchJobRepository:
}
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.cancel_requested.is_(False),
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
@@ -153,6 +153,16 @@ class DeepResearchSourceRepository:
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
if not source_rows:
# Collector persist marks rows selected, but an older session
# may only have unselected harvest rows. Cloning nothing would
# make 「选择结构再生成」 write with no materials.
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == source_session_id,
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
id_map: dict[str, str] = {}
cloned_rows: list[DeepResearchSourceRow] = []
@@ -191,14 +191,14 @@ class RunManager:
action: "interrupt" keeps checkpoint, "rollback" reverts to pre-run state.
Sets the abort event with the action reason and cancels the asyncio task.
Returns ``True`` if the run was in-flight and cancellation was initiated.
Returns ``True`` if cancellation was initiated or the run was already stopped.
"""
async with self._lock:
record = self._runs.get(run_id)
if record is None:
return False
return True
if record.status not in (RunStatus.pending, RunStatus.running):
return False
return True
record.abort_action = action
record.abort_event.set()
if record.task is not None and not record.task.done():
@@ -1389,17 +1389,40 @@ class ScheduledTaskService:
}
@staticmethod
def _build_conversion_prompt(skeleton_json: str, source: str, embedded_json: Any | None = None) -> str:
def _build_conversion_prompt(
skeleton_json: str,
source: str,
embedded_json: Any | None = None,
instructions: str | None = None,
) -> str:
parts = [
"请把下面的【源数据】转换为【目标结构】要求的 JSON。",
"请把下面的【源数据】转换成【目标结构】要求的 JSON,用于填充一个 HTML 展示模板。",
"模板只是一层**样式外壳**:它的字段名固定(页面脚本按字段名读取,绝不能改),"
"但每个字段的**含义要按源数据的主题重新解释**——把主题内容语义映射进各分区,"
"例如:总览指标填主题的核心数字;分组/区域类列表填主题的分区、品类或阵营;"
"组织/单位类列表填主题的机构、厂商或层级体系;装备/条目类列表填主题的产品、"
"技术或关键要素;事件/项目类列表填主题的时间线事项。内容必须全部来自【源数据】"
"的主题,不要被模板原有示例的领域限制住。",
"源数据可能是一段 JSON、一段纯文本、或包含 JSON 的文本——"
"其中若有结构化 JSON 以 JSON 内容为准;若是纯文本请理解其语义后据实填充;"
"源里没有对应内容的字段按其类型留空(\"\" / [] / null),不要臆造。",
"若下面的【任务要求】里带有对页面的明确修改指示(如“标题改为…”“只更新某部分”),"
"优先满足这些指示,其余字段可留空(页面将沿用模板现有内容);"
"没有指示到的部分按源数据自由发挥,尽量把各分区填满。",
"字段名必须与【目标结构】逐字一致(包括中文键名,原样复制),"
"严禁自造、翻译或改写字段名;数组元素的字段集合必须与样例元素相同,"
"源里没有的字段按类型留空即可,绝不能用自造的字段名顶替,也不要照抄任何示例值。",
"另外请在输出的 JSON 里额外携带两个顶层字符串字段:「页面标题」(按主题拟一个"
"页面大标题;若任务要求里明确给了标题,就用它)和「页面副标题」(一句话副标题)。",
"",
"【目标结构】(展示了完整字段与类型,其中数组只示意 1 个样例元素,实际请按源数据填充全部元素):",
"【目标结构】(字段与类型的空占位骨架,数组只示意 1 个样例元素,实际按源数据填充全部元素):",
"```json\n" + skeleton_json + "\n```",
"",
]
if instructions and instructions.strip():
parts.append("【任务要求】(用户对本次页面的要求/修改指示,优先级最高):")
parts.append(instructions.strip())
parts.append("")
if embedded_json is not None:
# 已从源数据里识别出结构化 JSON,单独标注帮助(尤其弱)模型对齐。
parts.append("【从源数据中识别到的结构化数据】(优先以此为准):")
@@ -1410,14 +1433,14 @@ class ScheduledTaskService:
parts.append("【源数据】:")
parts.append(source)
parts.append("")
parts.append("只输出一个 ```json 代码块,内含填充好的完整目标 JSON,不要任何解释文字。")
parts.append("只输出一个 ```json 代码块,内含填充好的完整目标 JSON(含「页面标题」「页面副标题」),不要任何解释文字。")
return "\n".join(parts)
@staticmethod
def _build_retry_prompt(base_prompt: str, errors: list[str], gaps: list[str]) -> str:
def _build_retry_prompt(base_prompt: str, problems: list[str], gaps: list[str]) -> str:
lines = [base_prompt, "\n\n上一轮转换结果未通过校验,请严格修正后重新输出**完整**的目标 JSON:"]
if errors:
lines.append("- 结构错误:" + ";".join(errors[:12]))
if problems:
lines.append("- " + ";".join(problems[:12]))
if gaps:
lines.append("- 字段缺失(请补齐):" + "、".join(gaps[:30]))
return "\n".join(lines)
@@ -1474,36 +1497,51 @@ class ScheduledTaskService:
model_name: str | None,
soul: str | None,
run_id: str,
instructions: str | None = None,
) -> dict[str, Any]:
"""把源数据(JSON/文本/含JSON文本)转换成模板的 DATA 结构,校验+回喂重试。
成功返回校验通过的对象;全部尝试失败抛 :class:`TemplateFillError`。
通过(结构+字段对齐)返回校验过的对象;重试耗尽时**尽力而为**——只要解析
出过 JSON 对象就返回最后一个(缺顶层键/字段错位的分支由随后的
``merge_with_reference`` 回退模板默认数据,抽多少用多少);只有全程没解析
出 JSON 才抛 :class:`TemplateFillError`(由调用方兜底为默认数据,不让
run 失败)。
"""
skeleton = template_fill.schema_skeleton_json(template_id)
# 源数据可能是纯 JSON / 纯文本 / 含 JSON 的文本:先尽力抽出内嵌 JSON 单独标注。
embedded_json = template_fill.extract_embedded_json(source)
base_prompt = self._build_conversion_prompt(skeleton, source, embedded_json)
last_errors: list[str] = []
base_prompt = self._build_conversion_prompt(skeleton, source, embedded_json, instructions=instructions)
last_problems: list[str] = []
prompt = base_prompt
candidate: dict[str, Any] | None = None
for attempt in range(_MAX_TEMPLATE_FILL_RETRIES + 1):
raw = await self._invoke_conversion_model(model_name, soul, prompt)
candidate = html_page_utils.extract_json_object(raw)
if candidate is None:
last_errors = ["模型未返回可解析的 JSON 对象"]
parsed = html_page_utils.extract_json_object(raw)
if parsed is None:
last_problems = ["模型未返回可解析的 JSON 对象"]
else:
errors = template_fill.validate_data(template_id, candidate)
if not errors:
candidate = parsed # 记住最后一个可解析结果,供重试耗尽时尽力返回
errors = [f"结构错误:{e}" for e in template_fill.validate_data(template_id, candidate)]
mismatches = [f"字段错位:{m}" for m in template_fill.shape_mismatches(template_id, candidate)]
if not errors and not mismatches:
return candidate
last_errors = errors
last_problems = errors + mismatches
gaps = template_fill.field_coverage_gaps(template_id, candidate)
prompt = self._build_retry_prompt(base_prompt, errors, gaps)
prompt = self._build_retry_prompt(base_prompt, last_problems, gaps)
logger.info(
"模板填充[run=%s]第 %d 次校验未通过:%s",
run_id,
attempt + 1,
";".join(last_errors[:5]),
";".join(last_problems[:5]),
)
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_errors[:8]))
if candidate is not None:
logger.warning(
"模板填充[run=%s]重试耗尽仍未通过校验,尽力返回待兜底:%s",
run_id,
";".join(last_problems[:5]),
)
return candidate
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_problems[:8]))
async def _finalize_template_html(
self,
@@ -1647,14 +1685,21 @@ class ScheduledTaskService:
# HTML 数据来源:抽取(开关开 + 有数据)→ 转换填充;否则用模板默认数据(子页面齐全)。
source = (data_text or "").strip()
used_default = True
page_title: str | None = None
page_subtitle: str | None = None
if do_extract and source:
# 转换阶段同样带上任务所选智能体:SOUL 提供领域口径,未显式选模型时
# 兜底用该智能体配置的模型(与模板填充模式一致)。
soul, conv_model = await self._agent_soul_and_model(task, model_name)
# 任务的描述/prompt 作为「任务要求」注入:用户对页面的修改指示(如
# "标题改为…"、"只更新某部分")优先级最高,没指示就按检索内容自由发挥。
instructions = "\n".join(part for part in (description, prompt) if part)
try:
converted = await self._convert_to_template_data(
tpl_id, source, model_name=conv_model, soul=soul, run_id=run_id
tpl_id, source, model_name=conv_model, soul=soul, run_id=run_id, instructions=instructions or None
)
# 页面标题/副标题是额外键,摘出来替换页头,不进 DATA。
page_title, page_subtitle = template_fill.pop_page_titles(converted)
# 子页面没抽到有效内容的,回退模板原有数据,避免子页面空白。
data_obj = template_fill.merge_with_reference(tpl_id, converted)
content = f"已检索数据并套用模板「{spec.name}」(空子页面回退原数据,含全部子页面)"
@@ -1672,7 +1717,9 @@ class ScheduledTaskService:
)
try:
html = template_fill.fill_template(tpl_id, data_obj, update_date=local_time.date())
html = template_fill.fill_template(
tpl_id, data_obj, update_date=local_time.date(), title=page_title, subtitle=page_subtitle
)
except template_fill.TemplateError as exc:
raise HtmlGenerationError(f"模板填充失败:{exc}") from exc
return await self._finalize_template_html(
@@ -1701,8 +1748,9 @@ class ScheduledTaskService:
Pipeline: 取源 JSON(任务 prompt) → 用内置「模板数据转换器」agent 的 SOUL +
模型把它映射成目标结构 → 结构校验(不过则回喂错误重试) → 填充模板 → 归档
``index.html`` + 持久化 ``scheduled_html_pages`` → 返回 ``format: "html"``
结果(复用现有 HTML 预览)。任一步彻底失败抛 :class:`TemplateFillError`,
run 记为 failed。
结果(复用现有 HTML 预览)。转换质量问题一律**不失败**:抽不到/抽错的分支由
``merge_with_reference`` 回退模板原有数据,转换彻底失败则整页用模板默认数据;
只有模板文件本身缺失/损坏才抛 :class:`TemplateFillError` 让 run 失败。
"""
template_id = _task_template_id(task)
spec = template_fill.get_template(template_id) if template_id else None
@@ -1726,6 +1774,8 @@ class ScheduledTaskService:
used_default=True,
)
page_title: str | None = None
page_subtitle: str | None = None
if not source:
# 未提供源数据 → 用模板自带默认数据(不调模型、不校验)。
data_obj = template_fill.reference_data(template_id)
@@ -1740,18 +1790,33 @@ class ScheduledTaskService:
model_name = str(tcfg.get("model_name") or "").strip() or None
soul, model_name = await self._agent_soul_and_model(task, model_name)
converted = await self._convert_to_template_data(
template_id, source, model_name=model_name, soul=soul, run_id=run_id
)
# 子页面没抽到有效内容的,回退模板原有数据,避免子页面空白。
data_obj = template_fill.merge_with_reference(template_id, converted)
content = f"已根据源数据填充模板「{spec.name}」生成页面"
used_default = False
try:
converted = await self._convert_to_template_data(
template_id, source, model_name=model_name, soul=soul, run_id=run_id
)
except TemplateFillError:
# 转换彻底失败(全程没解析出 JSON)→ 也不让 run 失败:用模板默认
# 数据出页,与 require_html 模板参考路径的兜底一致。
logger.warning(
"模板填充转换失败,回退默认数据 run=%s tpl=%s", run_id, template_id, exc_info=True
)
data_obj = template_fill.reference_data(template_id)
content = f"数据转换未通过,已用模板「{spec.name}」的默认数据生成页面(含全部子页面)"
used_default = True
else:
# 页面标题/副标题是额外键,摘出来替换页头,不进 DATA。
page_title, page_subtitle = template_fill.pop_page_titles(converted)
# 抽不到/抽错的子页面与字段回退模板原有数据,避免子页面空白或 undefined。
data_obj = template_fill.merge_with_reference(template_id, converted)
content = f"已根据源数据填充模板「{spec.name}」生成页面(抽取不到的部分沿用模板原有数据)"
used_default = False
task_tz = ZoneInfo(task.get("timezone") or "UTC")
local_time = scheduled_for.astimezone(task_tz)
try:
html = template_fill.fill_template(template_id, data_obj, update_date=local_time.date())
html = template_fill.fill_template(
template_id, data_obj, update_date=local_time.date(), title=page_title, subtitle=page_subtitle
)
except template_fill.TemplateError as exc:
raise TemplateFillError(f"模板填充失败:{exc}") from exc
return await self._finalize_template_html(
@@ -17,7 +17,8 @@ HTML 模板需要的 ``DATA`` 结构后,本模块负责把数据**填充**进模
* **harness 层、零 app.* 依赖**:可被调度器(harness)与 Gateway 路由(app)共同 import。
公开入口:``list_templates`` / ``get_template`` / ``reference_data`` /
``schema_skeleton_json`` / ``validate_data`` / ``field_coverage_gaps`` / ``fill_template``。
``schema_skeleton_json`` / ``validate_data`` / ``field_coverage_gaps`` /
``shape_mismatches`` / ``fill_template``。
"""
from __future__ import annotations
@@ -406,25 +407,33 @@ def reference_data(template_id: str) -> dict[str, Any]:
def _skeleton(value: Any, *, list_sample: int = 1, depth: int = 0, max_depth: int = 6) -> Any:
"""把一份具体数据压成「结构骨架」:保留全部字段与类型,每个数组只留前 N 个样例。
"""把一份具体数据压成「空占位骨架」:保留全部字段名与类型,值一律抹空。
数组元素同构,留 1 个样例即可暴露该元素的完整字段集,既完整又紧凑——正好做大模型
的「目标格式说明」。
模板只是样式外壳,**示例值(模板自带的模拟内容)绝不能进入转换 prompt**——
否则大模型会照抄示例内容、或被其领域(如台海防卫)限制住,不肯把用户主题的
内容映射进来。叶子按类型给空占位:字符串→\"\"、数字→0、布尔→false、
其它→null;数组元素同构,留 1 个抹空样例即可暴露该元素的完整字段集。
"""
if depth >= max_depth:
return "…"
return None
if isinstance(value, dict):
return {k: _skeleton(v, list_sample=list_sample, depth=depth + 1, max_depth=max_depth) for k, v in value.items()}
if isinstance(value, list):
if not value:
return []
return [_skeleton(v, list_sample=list_sample, depth=depth + 1, max_depth=max_depth) for v in value[:list_sample]]
return value
if isinstance(value, bool):
return False
if isinstance(value, (int, float)):
return 0
if isinstance(value, str):
return ""
return None
@lru_cache(maxsize=16)
def schema_skeleton_json(template_id: str) -> str:
"""目标格式的结构骨架(JSON 字符串),注入到大模型的转换 prompt 里。"""
"""目标格式的空占位骨架(JSON 字符串),注入到大模型的转换 prompt 里。"""
skeleton = _skeleton(reference_data(template_id))
return json.dumps(skeleton, ensure_ascii=False, indent=1)
@@ -509,11 +518,134 @@ def _has_content(value: Any) -> bool:
return True # 数字 / 布尔 / 其它标量
# 元素字段中至少要有这个比例能在模板参考数据的元素字段里找到,否则视为「自造字段」。
_LIST_FIELD_OVERLAP_MIN = 0.5
# 元素字段至少要覆盖模板参考元素字段的这个比例:低于它意味着大部分展示字段缺失
# (卡片大面积空白/undefined),不如整支回退模板原数据。
_LIST_FIELD_RECALL_MIN = 0.4
def _list_misaligned(got: Any, ref_val: Any) -> bool:
"""非空列表与模板参考列表是否「无法有效渲染」。
顶层校验挡不住这种情况:转换结果键齐全、类型也对,但列表元素的**字段**不对,
填进模板后 JS 读不到字段,页面满屏 ``undefined``。判定(满足其一即错位):
* 模板元素是对象而转换给了非对象(字符串/数字读不出属性);
* 元素字段里能对上模板的比例过低(:data:`_LIST_FIELD_OVERLAP_MIN`,
自造字段名占主导);
* 元素字段对模板参考字段的**覆盖率**过低(:data:`_LIST_FIELD_RECALL_MIN`,
只填了零星几个字段,展示字段大面积缺失)。
空列表不算错位(交由「无内容回退」处理)。
"""
if not isinstance(got, list) or not got:
return False
ref_fields = _element_fields(ref_val)
if not ref_fields:
return False # 模板元素本身没有对象字段,无从比对
ref_dicts = [item for item in ref_val if isinstance(item, dict)]
got_dicts = [item for item in got if isinstance(item, dict)]
if ref_dicts and not got_dicts:
return True # 模板元素是对象,转换给了字符串/数字 → 必然读不到字段
if not got_dicts:
return False
got_fields: set[str] = set()
for item in got_dicts:
got_fields.update(item.keys())
if not got_fields:
return False
overlap = got_fields & ref_fields
if len(overlap) / len(got_fields) < _LIST_FIELD_OVERLAP_MIN:
return True # 自造字段名占主导
return len(overlap) / len(ref_fields) < _LIST_FIELD_RECALL_MIN # 覆盖率过低
def _mismatch_reason(got: Any, ref_val: Any) -> str:
"""错位原因的人话描述(回喂重试用),与 :func:`_list_misaligned` 同判定。"""
ref_fields = _element_fields(ref_val)
got_fields: set[str] = set()
for item in got or []:
if isinstance(item, dict):
got_fields.update(item.keys())
overlap = got_fields & ref_fields
if got_fields and len(overlap) / len(got_fields) < _LIST_FIELD_OVERLAP_MIN:
return "的元素字段名与模板不一致(疑似自造字段名,必须原样沿用目标结构的字段名)"
return "的元素字段覆盖过低(样例元素的字段大部分缺失;缺数据的字段按类型留空补齐,不要省略字段)"
def shape_mismatches(template_id: str, obj: Any, *, max_items: int = 20) -> list[str]:
"""硬反馈:列出非空列表分支里「无法有效渲染」的项(供回喂重试)。
与 :func:`field_coverage_gaps`(软反馈,字段没填全)不同,本函数抓的是
**填了也渲染不出来**的情况:列表有内容但元素字段自造(对不上模板)或覆盖
过低(展示字段大面积缺失) → 渲染必然大面积 undefined,必须回喂重试。
"""
out: list[str] = []
if not isinstance(obj, dict):
return out
ref = reference_data(template_id)
for key, ref_val in ref.items():
got = obj.get(key)
if isinstance(ref_val, list) and _list_misaligned(got, ref_val):
out.append(f"「{key}」{_mismatch_reason(got, ref_val)}")
if len(out) >= max_items:
break
return out
def _ref_element_types(ref_val: list) -> dict[str, Any]:
"""参考列表各字段的类型样本(取第一个携带该字段的元素)。"""
types: dict[str, Any] = {}
for item in ref_val:
if isinstance(item, dict):
for key, value in item.items():
types.setdefault(key, value)
return types
def _blank_like(sample: Any) -> Any:
"""按参考类型给出「空值」:字符串→空串、列表→空数组、对象→空对象、其它→null。"""
if isinstance(sample, str):
return ""
if isinstance(sample, list):
return []
if isinstance(sample, dict):
return {}
return None
def _fill_missing_fields(got: list, ref_val: list) -> list:
"""给保留的转换列表逐元素补齐模板元素字段(缺失的按参考类型给空值)。
模板 JS 常裸读 ``${item.字段}``(不经过空值兜底函数),字段缺失会直接把
``undefined`` 打到页面上;补齐成空值后渲染为空白,不再出现字面 undefined。
只补缺、不改已有值,多出来的自造键原样保留(JS 不读,无害)。
"""
types = _ref_element_types(ref_val)
if not types:
return got
out: list[Any] = []
for item in got:
if isinstance(item, dict):
filled = dict(item)
for key, sample in types.items():
if key not in filled:
filled[key] = _blank_like(sample)
out.append(filled)
else:
out.append(item)
return out
def _merge_fill(got: Any, ref: Any) -> Any:
"""用 ``got`` 覆盖,但 ``got`` 里「没有有效内容」的部分回退 ``ref``。
字典逐键递归(空分支回退);列表/标量有内容就整体保留 ``got``(列表不逐元素合并,
避免与原模板元素错配)。
字典逐键递归(空分支回退);列表有内容就整体保留 ``got``(不逐元素合并,避免与
原模板元素错配)——**例外**:列表元素无法有效渲染(:func:`_list_misaligned`,
自造字段名占主导/覆盖率过低/元素类型不符)时整支回退 ``ref``,否则填进模板
JS 读不到字段,页面满屏 undefined。保留的列表会逐元素补齐缺失字段的空值
(:func:`_fill_missing_fields`),防止局部缺字段裸读出 undefined。
"""
if not _has_content(got):
return ref
@@ -522,6 +654,10 @@ def _merge_fill(got: Any, ref: Any) -> Any:
for key, ref_val in ref.items():
out[key] = _merge_fill(got.get(key), ref_val)
return out
if isinstance(ref, list) and isinstance(got, list):
if _list_misaligned(got, ref):
return ref
return _fill_missing_fields(got, ref)
return got
@@ -541,21 +677,78 @@ def _dump_const(var: str, obj: Any) -> str:
return f"const {var}={body};"
def _escape_html(text: str) -> str:
return (
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
)
# 转换结果里约定的两个额外顶层键:页面大标题/副标题(填充时替换页头,不进 DATA)。
PAGE_TITLE_KEY = "页面标题"
PAGE_SUBTITLE_KEY = "页面副标题"
def pop_page_titles(obj: dict[str, Any]) -> tuple[str | None, str | None]:
"""取出并移除转换结果里的「页面标题/页面副标题」额外键。
模型按转换 prompt 的约定额外输出这两个顶层字符串键;它们不是模板 DATA 的
字段(校验本就忽略额外键),填充前摘出来用于替换页头,避免漏进 ``const DATA``。
"""
def _clean(value: Any) -> str | None:
if isinstance(value, str) and value.strip():
return value.strip()
return None
title = _clean(obj.pop(PAGE_TITLE_KEY, None))
subtitle = _clean(obj.pop(PAGE_SUBTITLE_KEY, None))
return title, subtitle
def fill_template(
template_id: str,
data_obj: Any,
*,
manifest_overrides: dict[str, Any] | None = None,
update_date: date | None = None,
title: str | None = None,
subtitle: str | None = None,
) -> str:
"""把 ``data_obj`` 填进模板的 ``const DATA=…`` 行,产出完整 HTML 文档。
可选地更新 MANIFEST 的「系统.更新时间」(``update_date``,默认不动) 以及任意
``manifest_overrides``(深合并到解析后的 MANIFEST,如改系统名称)。
``title``/``subtitle`` 来自转换结果里的「页面标题/页面副标题」:模板页头
(<title>、第一个 <h1>、其后紧跟的副标题 <p>)是写死的静态 HTML,不随 DATA
变化——给出新标题时替换这三处,并把 MANIFEST 的 系统.名称 一并同步;
不给则保持模板原文。
"""
spec = _require(template_id)
text = read_template_text(template_id)
if title:
overrides = dict(manifest_overrides or {})
system = overrides.get("系统")
system = dict(system) if isinstance(system, dict) else {}
system["名称"] = title
overrides["系统"] = system
manifest_overrides = overrides
escaped = _escape_html(title)
text = re.sub(r"<title>.*?</title>", lambda _m: f"<title>{escaped}</title>", text, count=1, flags=re.DOTALL)
text = re.sub(r"<h1>.*?</h1>", lambda _m: f"<h1>{escaped}</h1>", text, count=1, flags=re.DOTALL)
if subtitle:
escaped_sub = _escape_html(subtitle)
text = re.sub(
r"(<h1>.*?</h1>\s*<p>).*?(</p>)",
lambda m: f"{m.group(1)}{escaped_sub}{m.group(2)}",
text,
count=1,
flags=re.DOTALL,
)
data_re = _const_line_re(spec.data_var)
if data_re.search(text) is None:
raise TemplateError(f"模板 {template_id} 中未找到 const {spec.data_var}= 行")