feat(定时任务): 内置「模板数据转换器」按 HTML 模板填充 JSON 生成大屏

- 模板填充引擎 template_fill.py:模板注册表 + schema 骨架 + 结构校验 + 行级安全填充;
  模板自带 DATA 即 schema 来源。no.1 可转换填充(含 MANIFEST 更新时间刷新),
  no.2 暂为「只展示」(convertible=False) 原样输出已有数据
- 内置 agent template-json-builder(seed + SOUL),随 lifespan 注入,自动出现在
  定时任务「执行智能体」下拉
- 调度器新增模板填充执行分支 _run_template_fill_attempt:源 JSON→大模型转换→
  结构校验→失败回喂重试;空源/只展示模板直接用模板默认数据,绝不失败
- API:GET /api/scheduled-tasks/html-templates
- 前端:选中该 agent 后条件渲染「HTML模板」下拉 + .json 上传,只展示型模板隐藏 JSON 输入
- 文档同步(前后端 CLAUDE.md)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-23 18:00:00 +08:00
1 parent 1a2ef0a537
commit cd33aaeded
17 files changed
+1511 -57

No files matched your search

@@ -18,11 +18,12 @@ from zoneinfo import ZoneInfo
from croniter import croniter
from langgraph.checkpoint.base import empty_checkpoint
from deerflow.config.agents_config import AGENT_ID_PATTERN
from deerflow.config.agents_config import AGENT_ID_PATTERN, load_agent_config, load_agent_soul
from deerflow.config.paths import get_paths
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.scheduler import html_page as html_page_utils
from deerflow.runtime.scheduler import template_fill
from deerflow.runtime.scheduler.element import ElementDeliveryClient
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.utils.time import now_iso
@@ -47,6 +48,10 @@ _MAX_INLINE_RESULT_HTML = 60000
# this threshold — it is enforced silently by the service.
_MARKDOWN_MIN_CHARS = 100
# 模板填充模式:转换结果未通过结构校验时,把错误回喂大模型自动修正重试的最大次数
# (总尝试次数 = 1 + 该值)。
_MAX_TEMPLATE_FILL_RETRIES = 2
# Instruction appended to the prompt when ``require_markdown`` is on. It only
# asks the model to produce a ``.md`` file; the length requirement is enforced
# in code (not surfaced to the model).
@@ -81,6 +86,13 @@ class HtmlGenerationError(Exception):
counts as successful when it is a full, previewable HTML document."""
class TemplateFillError(Exception):
"""Raised when a 模板填充 run could not produce a valid template payload —
the source JSON was empty, the model never returned parseable JSON, or the
converted data failed structure validation after all retries. The run is
marked failed so the user sees why (no template page is archived)."""
class DuplicateFavoriteError(Exception):
"""A user already bookmarked this page. Surfaced as HTTP 409 by the router
so the same page cannot be favorited twice."""
@@ -246,6 +258,23 @@ def _task_requires_html(task: dict[str, Any]) -> bool:
return ctx.get("task_kind") == "html_page"
def _task_template_id(task: dict[str, Any]) -> str | None:
"""The HTML template a 模板填充 task targets, or ``None`` for a normal task.
Stored at ``execution_context.template_config.template_id``. When set, the
run converts the source JSON into that template's ``DATA`` structure and
fills it — a distinct mode from the free-form HTML render pipeline.
"""
ctx = task.get("execution_context")
if not isinstance(ctx, dict):
return None
cfg = ctx.get("template_config")
if not isinstance(cfg, dict):
return None
template_id = cfg.get("template_id")
return str(template_id) if template_id else None
def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
"""True when the run produced a ``.md`` file whose text exceeds the minimum.
@@ -980,6 +1009,12 @@ class ScheduledTaskService:
needed, and old ``task_kind == "html_page"`` tasks still resolve to the
HTML pipeline via :func:`_task_requires_html`.
"""
# 模板填充模式优先:选中了 HTML 模板就走「转换 JSON → 填充固定模板」分支,
# 与「检索数据 → 自由渲染 HTML」(require_html) 是两套不同流程。
if _task_template_id(task) is not None:
return await self._run_template_fill_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
if _task_requires_html(task):
return await self._run_html_page_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
@@ -1339,6 +1374,196 @@ class ScheduledTaskService:
"created_at": datetime.now(UTC).isoformat(),
}
@staticmethod
def _build_conversion_prompt(skeleton_json: str, source: str) -> str:
return (
"请把下面的【源数据】转换为【目标结构】要求的 JSON。\n\n"
"【目标结构】(展示了完整字段与类型,其中数组只示意 1 个样例元素,实际请按源数据填充全部元素):\n"
"```json\n" + skeleton_json + "\n```\n\n"
"【源数据】:\n" + source + "\n\n"
"只输出一个 ```json 代码块,内含填充好的完整目标 JSON,不要任何解释文字。"
)
@staticmethod
def _build_retry_prompt(base_prompt: str, errors: list[str], gaps: list[str]) -> str:
lines = [base_prompt, "\n\n上一轮转换结果未通过校验,请严格修正后重新输出**完整**的目标 JSON:"]
if errors:
lines.append("- 结构错误:" + ";".join(errors[:12]))
if gaps:
lines.append("- 字段缺失(请补齐):" + "、".join(gaps[:30]))
return "\n".join(lines)
async def _invoke_conversion_model(self, model_name: str | None, soul: str | None, user_prompt: str) -> str:
"""Invoke the conversion model once (SOUL as system, prompt as human).
Thinking is disabled for stable JSON output. Returns the model's text.
"""
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
messages: list[Any] = []
if soul:
messages.append(SystemMessage(content=soul))
messages.append(HumanMessage(content=user_prompt))
response = await model.ainvoke(messages)
content = getattr(response, "content", response)
if isinstance(content, str):
return content
if isinstance(content, list):
parts = [part.get("text", "") if isinstance(part, dict) else str(part) for part in content]
return "\n".join(p for p in parts if p)
return str(content)
async def _run_template_fill_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""模板填充模式:把源 JSON 转换成所选模板的 DATA 结构,校验后填充产出 HTML。
Pipeline: 取源 JSON(任务 prompt) → 用内置「模板数据转换器」agent 的 SOUL +
模型把它映射成目标结构 → 结构校验(不过则回喂错误重试) → 填充模板 → 归档
``index.html`` + 持久化 ``scheduled_html_pages`` → 返回 ``format: "html"``
结果(复用现有 HTML 预览)。任一步彻底失败抛 :class:`TemplateFillError`,
run 记为 failed。
"""
template_id = _task_template_id(task)
spec = template_fill.get_template(template_id) if template_id else None
if spec is None:
raise TemplateFillError(f"未知的 HTML 模板:{template_id!r}")
source = str(task.get("prompt") or "").strip()
used_default = False
data_obj: dict[str, Any] | None = None
# 「只展示」模板(convertible=False):暂不转换,原样输出模板已有的 HTML。
if not spec.convertible:
html = template_fill.read_template_text(template_id)
used_default = True
elif not source:
# 未提供源数据 → 直接展示模板自带的默认数据(不调模型、不校验)。
data_obj = template_fill.reference_data(template_id)
used_default = True
else:
ctx = task.get("execution_context") or {}
tcfg = ctx.get("template_config") if isinstance(ctx, dict) else {}
tcfg = tcfg if isinstance(tcfg, dict) else {}
# 解析选中的内置 agent 的 SOUL + 模型(template_config.model_name 优先,其次 agent.model)。
agent_ref = task.get("execution_agent_name")
agent_id = await self._resolve_agent_id(agent_ref, task["user_id"]) if agent_ref else None
soul = load_agent_soul(agent_id) if agent_id else None
model_name = str(tcfg.get("model_name") or "").strip() or None
if not model_name and agent_id:
try:
agent_cfg = load_agent_config(agent_id)
model_name = (agent_cfg.model if agent_cfg else None) or None
except Exception:
model_name = None
skeleton = template_fill.schema_skeleton_json(template_id)
base_prompt = self._build_conversion_prompt(skeleton, source)
# 转换 → 校验 → 失败回喂重试。
last_errors: list[str] = []
prompt = base_prompt
for attempt in range(_MAX_TEMPLATE_FILL_RETRIES + 1):
raw = await self._invoke_conversion_model(model_name, soul, prompt)
candidate = html_page_utils.extract_json_object(raw)
if candidate is None:
last_errors = ["模型未返回可解析的 JSON 对象"]
else:
errors = template_fill.validate_data(template_id, candidate)
if not errors:
data_obj = candidate
break
last_errors = errors
gaps = template_fill.field_coverage_gaps(template_id, candidate)
prompt = self._build_retry_prompt(base_prompt, errors, gaps)
logger.info(
"模板填充[run=%s]第 %d 次校验未通过:%s",
run_id,
attempt + 1,
";".join(last_errors[:5]),
)
if data_obj is None:
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_errors[:8]))
# 填充模板(顺便把 MANIFEST 更新时间刷成本次计划日期)。「只展示」模板上面已
# 把原始 HTML 放进 html,这里跳过填充。
task_tz = ZoneInfo(task.get("timezone") or "UTC")
local_time = scheduled_for.astimezone(task_tz)
if data_obj is not None:
try:
html = template_fill.fill_template(template_id, data_obj, update_date=local_time.date())
except template_fill.TemplateError as exc:
raise TemplateFillError(f"模板填充失败:{exc}") from exc
# 归档 index.html。
archive_path = self._archive_html_index(run_id, html)
title = str(task.get("name") or spec.name).strip()
files: list[dict[str, Any]] = []
if archive_path is not None:
files.append(
{
"name": "index.html",
"path": str(archive_path),
"virtual_path": "/mnt/user-data/outputs/index.html",
"mime_type": "text/html",
"size": archive_path.stat().st_size if archive_path.exists() else None,
"kind": "html",
}
)
data_snapshot = {"template_id": template_id, "source_chars": len(source), "used_default": used_default}
prompt_snapshot = {"template_id": template_id, "used_default": used_default}
# 持久化页面资产(best-effort,失败不影响 run 成功)。
store = self._html_store
if store is not None:
try:
await store.create_page(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"run_id": run_id,
"user_id": task["user_id"],
"title": title,
"html_content": html,
"html_file_path": str(archive_path) if archive_path else None,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"reference_favorite_id": None,
"style_summary": None,
"layout_summary": None,
}
)
except Exception:
logger.warning("Failed to persist scheduled_html_pages row for template-fill run %s", run_id, exc_info=True)
inline_html = html if len(html) <= _MAX_INLINE_RESULT_HTML else None
return {
"title": f"{title} - {local_time.strftime('%Y-%m-%d %H:%M')}",
"content": (
f"已展示模板「{spec.name}」的现有数据"
if not spec.convertible
else f"未提供源数据,已用模板「{spec.name}」的默认数据生成页面"
if used_default
else f"已根据源数据填充模板「{spec.name}」生成页面"
),
"format": "html",
"html": inline_html,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"files": files,
"created_at": datetime.now(UTC).isoformat(),
}
async def _run_agent_attempt(
self,
task: dict[str, Any],