feat(定时任务): 内置「模板数据转换器」按 HTML 模板填充 JSON 生成大屏
- 模板填充引擎 template_fill.py:模板注册表 + schema 骨架 + 结构校验 + 行级安全填充; 模板自带 DATA 即 schema 来源。no.1 可转换填充(含 MANIFEST 更新时间刷新), no.2 暂为「只展示」(convertible=False) 原样输出已有数据 - 内置 agent template-json-builder(seed + SOUL),随 lifespan 注入,自动出现在 定时任务「执行智能体」下拉 - 调度器新增模板填充执行分支 _run_template_fill_attempt:源 JSON→大模型转换→ 结构校验→失败回喂重试;空源/只展示模板直接用模板默认数据,绝不失败 - API:GET /api/scheduled-tasks/html-templates - 前端:选中该 agent 后条件渲染「HTML模板」下拉 + .json 上传,只展示型模板隐藏 JSON 输入 - 文档同步(前后端 CLAUDE.md) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1a2ef0a537
commit
cd33aaeded
17 files changed
+1511
-57
No files matched your search
+226
-1
@@ -18,11 +18,12 @@ from zoneinfo import ZoneInfo
|
||||
from croniter import croniter
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
|
||||
from deerflow.config.agents_config import AGENT_ID_PATTERN
|
||||
from deerflow.config.agents_config import AGENT_ID_PATTERN, load_agent_config, load_agent_soul
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.scheduler import html_page as html_page_utils
|
||||
from deerflow.runtime.scheduler import template_fill
|
||||
from deerflow.runtime.scheduler.element import ElementDeliveryClient
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.utils.time import now_iso
|
||||
@@ -47,6 +48,10 @@ _MAX_INLINE_RESULT_HTML = 60000
|
||||
# this threshold — it is enforced silently by the service.
|
||||
_MARKDOWN_MIN_CHARS = 100
|
||||
|
||||
# 模板填充模式:转换结果未通过结构校验时,把错误回喂大模型自动修正重试的最大次数
|
||||
# (总尝试次数 = 1 + 该值)。
|
||||
_MAX_TEMPLATE_FILL_RETRIES = 2
|
||||
|
||||
# Instruction appended to the prompt when ``require_markdown`` is on. It only
|
||||
# asks the model to produce a ``.md`` file; the length requirement is enforced
|
||||
# in code (not surfaced to the model).
|
||||
@@ -81,6 +86,13 @@ class HtmlGenerationError(Exception):
|
||||
counts as successful when it is a full, previewable HTML document."""
|
||||
|
||||
|
||||
class TemplateFillError(Exception):
|
||||
"""Raised when a 模板填充 run could not produce a valid template payload —
|
||||
the source JSON was empty, the model never returned parseable JSON, or the
|
||||
converted data failed structure validation after all retries. The run is
|
||||
marked failed so the user sees why (no template page is archived)."""
|
||||
|
||||
|
||||
class DuplicateFavoriteError(Exception):
|
||||
"""A user already bookmarked this page. Surfaced as HTTP 409 by the router
|
||||
so the same page cannot be favorited twice."""
|
||||
@@ -246,6 +258,23 @@ def _task_requires_html(task: dict[str, Any]) -> bool:
|
||||
return ctx.get("task_kind") == "html_page"
|
||||
|
||||
|
||||
def _task_template_id(task: dict[str, Any]) -> str | None:
|
||||
"""The HTML template a 模板填充 task targets, or ``None`` for a normal task.
|
||||
|
||||
Stored at ``execution_context.template_config.template_id``. When set, the
|
||||
run converts the source JSON into that template's ``DATA`` structure and
|
||||
fills it — a distinct mode from the free-form HTML render pipeline.
|
||||
"""
|
||||
ctx = task.get("execution_context")
|
||||
if not isinstance(ctx, dict):
|
||||
return None
|
||||
cfg = ctx.get("template_config")
|
||||
if not isinstance(cfg, dict):
|
||||
return None
|
||||
template_id = cfg.get("template_id")
|
||||
return str(template_id) if template_id else None
|
||||
|
||||
|
||||
def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
|
||||
"""True when the run produced a ``.md`` file whose text exceeds the minimum.
|
||||
|
||||
@@ -980,6 +1009,12 @@ class ScheduledTaskService:
|
||||
needed, and old ``task_kind == "html_page"`` tasks still resolve to the
|
||||
HTML pipeline via :func:`_task_requires_html`.
|
||||
"""
|
||||
# 模板填充模式优先:选中了 HTML 模板就走「转换 JSON → 填充固定模板」分支,
|
||||
# 与「检索数据 → 自由渲染 HTML」(require_html) 是两套不同流程。
|
||||
if _task_template_id(task) is not None:
|
||||
return await self._run_template_fill_attempt(
|
||||
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
|
||||
)
|
||||
if _task_requires_html(task):
|
||||
return await self._run_html_page_attempt(
|
||||
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
|
||||
@@ -1339,6 +1374,196 @@ class ScheduledTaskService:
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_conversion_prompt(skeleton_json: str, source: str) -> str:
|
||||
return (
|
||||
"请把下面的【源数据】转换为【目标结构】要求的 JSON。\n\n"
|
||||
"【目标结构】(展示了完整字段与类型,其中数组只示意 1 个样例元素,实际请按源数据填充全部元素):\n"
|
||||
"```json\n" + skeleton_json + "\n```\n\n"
|
||||
"【源数据】:\n" + source + "\n\n"
|
||||
"只输出一个 ```json 代码块,内含填充好的完整目标 JSON,不要任何解释文字。"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_prompt(base_prompt: str, errors: list[str], gaps: list[str]) -> str:
|
||||
lines = [base_prompt, "\n\n上一轮转换结果未通过校验,请严格修正后重新输出**完整**的目标 JSON:"]
|
||||
if errors:
|
||||
lines.append("- 结构错误:" + ";".join(errors[:12]))
|
||||
if gaps:
|
||||
lines.append("- 字段缺失(请补齐):" + "、".join(gaps[:30]))
|
||||
return "\n".join(lines)
|
||||
|
||||
async def _invoke_conversion_model(self, model_name: str | None, soul: str | None, user_prompt: str) -> str:
|
||||
"""Invoke the conversion model once (SOUL as system, prompt as human).
|
||||
|
||||
Thinking is disabled for stable JSON output. Returns the model's text.
|
||||
"""
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
messages: list[Any] = []
|
||||
if soul:
|
||||
messages.append(SystemMessage(content=soul))
|
||||
messages.append(HumanMessage(content=user_prompt))
|
||||
response = await model.ainvoke(messages)
|
||||
content = getattr(response, "content", response)
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [part.get("text", "") if isinstance(part, dict) else str(part) for part in content]
|
||||
return "\n".join(p for p in parts if p)
|
||||
return str(content)
|
||||
|
||||
async def _run_template_fill_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""模板填充模式:把源 JSON 转换成所选模板的 DATA 结构,校验后填充产出 HTML。
|
||||
|
||||
Pipeline: 取源 JSON(任务 prompt) → 用内置「模板数据转换器」agent 的 SOUL +
|
||||
模型把它映射成目标结构 → 结构校验(不过则回喂错误重试) → 填充模板 → 归档
|
||||
``index.html`` + 持久化 ``scheduled_html_pages`` → 返回 ``format: "html"``
|
||||
结果(复用现有 HTML 预览)。任一步彻底失败抛 :class:`TemplateFillError`,
|
||||
run 记为 failed。
|
||||
"""
|
||||
template_id = _task_template_id(task)
|
||||
spec = template_fill.get_template(template_id) if template_id else None
|
||||
if spec is None:
|
||||
raise TemplateFillError(f"未知的 HTML 模板:{template_id!r}")
|
||||
|
||||
source = str(task.get("prompt") or "").strip()
|
||||
used_default = False
|
||||
data_obj: dict[str, Any] | None = None
|
||||
|
||||
# 「只展示」模板(convertible=False):暂不转换,原样输出模板已有的 HTML。
|
||||
if not spec.convertible:
|
||||
html = template_fill.read_template_text(template_id)
|
||||
used_default = True
|
||||
elif not source:
|
||||
# 未提供源数据 → 直接展示模板自带的默认数据(不调模型、不校验)。
|
||||
data_obj = template_fill.reference_data(template_id)
|
||||
used_default = True
|
||||
else:
|
||||
ctx = task.get("execution_context") or {}
|
||||
tcfg = ctx.get("template_config") if isinstance(ctx, dict) else {}
|
||||
tcfg = tcfg if isinstance(tcfg, dict) else {}
|
||||
|
||||
# 解析选中的内置 agent 的 SOUL + 模型(template_config.model_name 优先,其次 agent.model)。
|
||||
agent_ref = task.get("execution_agent_name")
|
||||
agent_id = await self._resolve_agent_id(agent_ref, task["user_id"]) if agent_ref else None
|
||||
soul = load_agent_soul(agent_id) if agent_id else None
|
||||
model_name = str(tcfg.get("model_name") or "").strip() or None
|
||||
if not model_name and agent_id:
|
||||
try:
|
||||
agent_cfg = load_agent_config(agent_id)
|
||||
model_name = (agent_cfg.model if agent_cfg else None) or None
|
||||
except Exception:
|
||||
model_name = None
|
||||
|
||||
skeleton = template_fill.schema_skeleton_json(template_id)
|
||||
base_prompt = self._build_conversion_prompt(skeleton, source)
|
||||
|
||||
# 转换 → 校验 → 失败回喂重试。
|
||||
last_errors: list[str] = []
|
||||
prompt = base_prompt
|
||||
for attempt in range(_MAX_TEMPLATE_FILL_RETRIES + 1):
|
||||
raw = await self._invoke_conversion_model(model_name, soul, prompt)
|
||||
candidate = html_page_utils.extract_json_object(raw)
|
||||
if candidate is None:
|
||||
last_errors = ["模型未返回可解析的 JSON 对象"]
|
||||
else:
|
||||
errors = template_fill.validate_data(template_id, candidate)
|
||||
if not errors:
|
||||
data_obj = candidate
|
||||
break
|
||||
last_errors = errors
|
||||
gaps = template_fill.field_coverage_gaps(template_id, candidate)
|
||||
prompt = self._build_retry_prompt(base_prompt, errors, gaps)
|
||||
logger.info(
|
||||
"模板填充[run=%s]第 %d 次校验未通过:%s",
|
||||
run_id,
|
||||
attempt + 1,
|
||||
";".join(last_errors[:5]),
|
||||
)
|
||||
if data_obj is None:
|
||||
raise TemplateFillError("转换结果未通过结构校验:" + ";".join(last_errors[:8]))
|
||||
|
||||
# 填充模板(顺便把 MANIFEST 更新时间刷成本次计划日期)。「只展示」模板上面已
|
||||
# 把原始 HTML 放进 html,这里跳过填充。
|
||||
task_tz = ZoneInfo(task.get("timezone") or "UTC")
|
||||
local_time = scheduled_for.astimezone(task_tz)
|
||||
if data_obj is not None:
|
||||
try:
|
||||
html = template_fill.fill_template(template_id, data_obj, update_date=local_time.date())
|
||||
except template_fill.TemplateError as exc:
|
||||
raise TemplateFillError(f"模板填充失败:{exc}") from exc
|
||||
|
||||
# 归档 index.html。
|
||||
archive_path = self._archive_html_index(run_id, html)
|
||||
title = str(task.get("name") or spec.name).strip()
|
||||
files: list[dict[str, Any]] = []
|
||||
if archive_path is not None:
|
||||
files.append(
|
||||
{
|
||||
"name": "index.html",
|
||||
"path": str(archive_path),
|
||||
"virtual_path": "/mnt/user-data/outputs/index.html",
|
||||
"mime_type": "text/html",
|
||||
"size": archive_path.stat().st_size if archive_path.exists() else None,
|
||||
"kind": "html",
|
||||
}
|
||||
)
|
||||
|
||||
data_snapshot = {"template_id": template_id, "source_chars": len(source), "used_default": used_default}
|
||||
prompt_snapshot = {"template_id": template_id, "used_default": used_default}
|
||||
|
||||
# 持久化页面资产(best-effort,失败不影响 run 成功)。
|
||||
store = self._html_store
|
||||
if store is not None:
|
||||
try:
|
||||
await store.create_page(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"run_id": run_id,
|
||||
"user_id": task["user_id"],
|
||||
"title": title,
|
||||
"html_content": html,
|
||||
"html_file_path": str(archive_path) if archive_path else None,
|
||||
"data_snapshot": data_snapshot,
|
||||
"prompt_snapshot": prompt_snapshot,
|
||||
"reference_favorite_id": None,
|
||||
"style_summary": None,
|
||||
"layout_summary": None,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist scheduled_html_pages row for template-fill run %s", run_id, exc_info=True)
|
||||
|
||||
inline_html = html if len(html) <= _MAX_INLINE_RESULT_HTML else None
|
||||
return {
|
||||
"title": f"{title} - {local_time.strftime('%Y-%m-%d %H:%M')}",
|
||||
"content": (
|
||||
f"已展示模板「{spec.name}」的现有数据"
|
||||
if not spec.convertible
|
||||
else f"未提供源数据,已用模板「{spec.name}」的默认数据生成页面"
|
||||
if used_default
|
||||
else f"已根据源数据填充模板「{spec.name}」生成页面"
|
||||
),
|
||||
"format": "html",
|
||||
"html": inline_html,
|
||||
"data_snapshot": data_snapshot,
|
||||
"prompt_snapshot": prompt_snapshot,
|
||||
"files": files,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
async def _run_agent_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
|
||||
Reference in new issue
Block a user