feat(scheduled-tasks): 新增 HTML 页面定时任务(生成/预览/收藏/技能)

后端:
- 新增 task_kind=html_page 任务类型,与普通定时任务并存,配置存 execution_context(无需迁移)
- 执行流水线:智能体检索数据 -> 选定模型渲染 HTML -> 清洗/校验 -> 失败自动修复 -> 完整性闸门(缺 doctype/html/head/body/title 或残留脚本即失败) -> 归档 index.html -> 落库
- 新增 scheduled_html_pages / html_page_favorites 表与 store;收藏去重(不可重复)、可删除、收藏时提取可复用风格
- HTML 预览接口(鉴权 + 公开)、收藏接口
- 渲染注入已启用的 HTML 技能(metadata.kind=html_page);全程使用用户所选模型,去除硬编码 GLM-5
- 定时任务跳过 TitleMiddleware,避免标题生成回退默认模型

前端:
- 创建/编辑弹窗增加任务类型与 HTML 配置(页面要求/限制/参考收藏页/模型下拉)
- 运行详情页与公开页 iframe 沙箱预览 HTML;收藏按钮(已收藏禁用)
- 参考收藏页可选可删

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-02 23:46:20 +08:00
1 parent e1e9be8daa
commit d9afaca51f
22 files changed
+1996 -107

No files matched your search

@@ -22,6 +22,7 @@ from deerflow.config.agents_config import AGENT_ID_PATTERN
from deerflow.config.paths import get_paths
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.scheduler import html_page as html_page_utils
from deerflow.runtime.scheduler.element import ElementDeliveryClient
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.utils.time import now_iso
@@ -36,6 +37,11 @@ _RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
# Max HTML length (chars) inlined into the run's ``result_json``. Beyond this
# the inline copy is dropped to stay under MySQL's 64 KB ``TEXT`` cap; the full
# document is always retrievable from the page store (LONGTEXT) + index.html.
_MAX_INLINE_RESULT_HTML = 60000
# Minimum length (in characters) the generated Markdown file's content must
# reach when a task has ``require_markdown`` enabled. The model is NOT told about
# this threshold — it is enforced silently by the service.
@@ -58,6 +64,22 @@ class MarkdownRequirementError(Exception):
"""
class HtmlGenerationError(Exception):
"""Raised when an ``html_page`` run did not produce a complete, valid HTML
document (missing doctype/html/head/body, empty body, or residual scripts
after the sanitize + repair passes). The run is marked failed — a page only
counts as successful when it is a full, previewable HTML document."""
class DuplicateFavoriteError(Exception):
"""A user already bookmarked this page. Surfaced as HTTP 409 by the router
so the same page cannot be favorited twice."""
def __init__(self, page_id: str) -> None:
self.page_id = page_id
super().__init__(f"Page {page_id!r} is already favorited")
class DuplicateTaskNameError(Exception):
"""A user already owns a scheduled task with the requested name.
@@ -835,7 +857,7 @@ class ScheduledTaskService:
runtime_thread_id = str(uuid.uuid4())
try:
result = await asyncio.wait_for(
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
self._run_task_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
)
except TimeoutError:
@@ -875,6 +897,333 @@ class ScheduledTaskService:
await self._deliver_element_result(task, result["content"])
return
async def _run_task_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Dispatch a single execution attempt by task kind.
``html_page`` tasks render an HTML document via GLM5; everything else
runs the regular lead-agent attempt. The kind lives in
``execution_context.task_kind`` so no schema migration is needed.
"""
ctx = task.get("execution_context")
kind = ctx.get("task_kind") if isinstance(ctx, dict) else None
if kind == "html_page":
return await self._run_html_page_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
return await self._run_agent_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
@staticmethod
def _html_config(task: dict[str, Any]) -> dict[str, Any]:
ctx = task.get("execution_context")
cfg = ctx.get("html_config") if isinstance(ctx, dict) else None
return cfg if isinstance(cfg, dict) else {}
async def _invoke_chat_model(self, model_name: str | None, prompt: str) -> str:
"""Invoke a chat model once and return its text content.
Used for the GLM5 HTML generation / repair / style-extraction passes.
Thinking is disabled to keep structured-JSON output stable.
"""
from langchain_core.messages import HumanMessage
from deerflow.models import create_chat_model
# No hardcoded model: use exactly the selected model; only when none was
# selected do we fall back to the system default (config models[0]).
model = create_chat_model(name=model_name or None, thinking_enabled=False)
response = await model.ainvoke([HumanMessage(content=prompt)])
content = getattr(response, "content", response)
if isinstance(content, str):
return content
if isinstance(content, list):
parts = [part.get("text", "") if isinstance(part, dict) else str(part) for part in content]
return "\n".join(p for p in parts if p)
return str(content)
async def _gather_html_data(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> str:
"""Run the lead agent to collect source data for the HTML page.
Reuses the regular agent-run primitives (with the task's search query
folded into the prompt) and returns the agent's final text, which feeds
the GLM5 render step as ``本次数据``. Returns an empty string if the
agent produced nothing.
"""
cfg = self._html_config(task)
page_prompt = str(cfg.get("page_prompt") or "").strip()
gather_instruction = (
"请运用你自身的检索/搜索等技能,围绕下述页面需求获取最新、真实的资料,"
"并整理成结构化的中文要点(可用列表/小标题),供后续生成 HTML 页面使用,本步骤不要生成 HTML:\n"
)
parts = [gather_instruction]
if page_prompt:
parts.append(f"页面需求:{page_prompt}")
base_prompt = str(task.get("prompt") or "").strip()
if base_prompt:
parts.append(f"补充说明:{base_prompt}")
gather_task = {**task, "prompt": "\n".join(parts)}
# Never apply the markdown gate to the gather pass.
gather_ctx = dict(task.get("execution_context") or {})
gather_ctx.pop("require_markdown", None)
gather_ctx.pop("task_kind", None)
gather_ctx.pop("html_config", None)
# Use the task's selected model for the data-gathering agent too, so the
# whole HTML pipeline honors ``html_config.model_name`` instead of
# silently falling back to the default model for this step.
gather_model = str(cfg.get("model_name") or "").strip()
if gather_model:
gather_ctx["model_name"] = gather_model
gather_task["execution_context"] = gather_ctx
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
record = await self._start_agent_run(gather_task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
text = await self._extract_latest_ai_text(runtime_thread_id)
return text or ""
def _collect_html_skill_instructions(self) -> str:
"""Concatenate the bodies of enabled skills marked for HTML pages.
A skill opts in via ``metadata.kind: html_page`` in its SKILL.md
frontmatter. Only enabled skills (per the skills settings the user
toggles) are injected, so the render step honors the user's skill
selection. Best-effort — any failure yields an empty string.
"""
try:
from deerflow.skills import get_or_new_skill_storage
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
except Exception:
logger.debug("Failed to load skills for HTML render injection", exc_info=True)
return ""
blocks: list[str] = []
for skill in skills:
try:
text = Path(skill.skill_file).read_text(encoding="utf-8")
except Exception:
continue
frontmatter, body = html_page_utils.parse_frontmatter(text)
meta = frontmatter.get("metadata") if isinstance(frontmatter, dict) else None
if not (isinstance(meta, dict) and str(meta.get("kind")) == "html_page"):
continue
body = body.strip()
if body:
blocks.append(f"### 技能:{skill.name}\n{body}")
return "\n\n".join(blocks)
def _archive_html_index(self, run_id: str, html: str) -> Path | None:
"""Write the generated HTML to the run's archive dir as ``index.html``."""
try:
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id / "files"
archive_dir.mkdir(parents=True, exist_ok=True)
target = archive_dir / "index.html"
target.write_text(html, encoding="utf-8")
return target
except Exception:
logger.warning("Failed to archive generated HTML for run %s", run_id, exc_info=True)
return None
async def _run_html_page_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Generate an HTML page for an ``html_page`` task.
Pipeline: gather data via the lead agent → optionally load a reference
favorite's style/layout → GLM5 render → sanitize → validate → (one
GLM5 repair pass if invalid) → best-effort screenshot → archive
``index.html`` → persist a ``scheduled_html_pages`` row → build the run
result payload (``format: "html"`` with the full HTML inline).
"""
cfg = self._html_config(task)
# The user's selected model only — no hardcoded GLM-5. Empty → None →
# system default (config models[0]).
model_name = str(cfg.get("model_name") or "").strip() or None
page_prompt = str(cfg.get("page_prompt") or "").strip()
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
reference_favorite_id = cfg.get("reference_favorite_id") or None
# 1) Gather source data (real, via the agent's search tools).
data_text = await self._gather_html_data(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
# 2) Load reference style/layout from a favorite if one was selected.
reference_style = None
reference_layout = None
if reference_favorite_id:
store = self._html_store
if store is not None:
try:
favorite = await store.get_favorite(str(reference_favorite_id))
except Exception:
favorite = None
if favorite is not None:
reference_style = favorite.get("reuse_prompt") or favorite.get("style_summary")
reference_layout = favorite.get("layout_summary")
# 3) Render with the chosen model, injecting any enabled HTML skills.
skill_instructions = self._collect_html_skill_instructions()
gen_prompt = html_page_utils.build_generation_prompt(
page_prompt=page_prompt,
negative_prompt=negative_prompt,
data=data_text,
reference_style=reference_style,
reference_layout=reference_layout,
skill_instructions=skill_instructions,
)
raw = await self._invoke_chat_model(model_name, gen_prompt)
parsed = html_page_utils.extract_json_object(raw) or {}
title = str(parsed.get("title") or task.get("name") or "生成页面").strip()
html = parsed.get("html")
if not isinstance(html, str) or "<" not in html:
# GLM5 may have replied with bare HTML instead of JSON — accept it.
if isinstance(raw, str) and ("<html" in raw.lower() or "<!doctype" in raw.lower()):
html = raw
else:
raise ValueError("GLM5 未返回有效的 HTML(JSON 解析失败)")
style_summary = str(parsed.get("style_summary") or "").strip() or None
layout_summary = str(parsed.get("layout_summary") or "").strip() or None
# 4) Sanitize + validate, with one repair pass on failure.
html, _removed = html_page_utils.sanitize_html(html)
errors = html_page_utils.validate_html(html)
if errors:
logger.info("Generated HTML failed validation (%s); attempting repair", errors)
repair_prompt = html_page_utils.build_repair_prompt(
html=html, errors=errors, requirements=page_prompt or task.get("prompt")
)
try:
repaired_raw = await self._invoke_chat_model(model_name, repair_prompt)
repaired = html_page_utils.extract_json_object(repaired_raw) or {}
fixed = repaired.get("fixed_html")
if isinstance(fixed, str) and "<" in fixed:
fixed_clean, _ = html_page_utils.sanitize_html(fixed)
if not html_page_utils.validate_html(fixed_clean):
html = fixed_clean
except Exception:
logger.warning("HTML repair pass failed; keeping sanitized original", exc_info=True)
# 4b) Completeness gate — a page only counts as successful when it is a
# full, valid HTML document. If it still fails validation after the
# repair pass, fail the run (no half-baked page is archived/persisted).
final_errors = html_page_utils.validate_html(html)
if final_errors:
raise HtmlGenerationError(
"HTML 生成未通过完整性校验:" + ";".join(final_errors) + "。请调整页面要求或重试。"
)
# 5) Best-effort screenshot QA (no-op without Playwright).
screenshot_path = None
try:
shot_target = get_paths().base_dir / "scheduled-task-runs" / run_id / "files" / "preview.png"
shot_target.parent.mkdir(parents=True, exist_ok=True)
screenshot_path = await html_page_utils.screenshot_html(html, shot_target)
except Exception:
screenshot_path = None
# 6) Archive index.html.
archive_path = self._archive_html_index(run_id, html)
files: list[dict[str, Any]] = []
if archive_path is not None:
files.append(
{
"name": "index.html",
"path": str(archive_path),
"virtual_path": "/mnt/user-data/outputs/index.html",
"mime_type": "text/html",
"size": archive_path.stat().st_size if archive_path.exists() else None,
"kind": "html",
}
)
if screenshot_path:
shot = Path(screenshot_path)
files.append(
{
"name": "preview.png",
"path": str(shot),
"mime_type": "image/png",
"size": shot.stat().st_size if shot.exists() else None,
"kind": "artifact",
}
)
data_snapshot = {"data_text": data_text[:8000] if isinstance(data_text, str) else ""}
prompt_snapshot = {
"page_prompt": page_prompt,
"negative_prompt": negative_prompt,
"reference_favorite_id": reference_favorite_id,
"model_name": model_name,
}
# 7) Persist the page asset (best-effort — failure must not fail the run).
store = self._html_store
if store is not None:
try:
await store.create_page(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"run_id": run_id,
"user_id": task["user_id"],
"title": title,
"html_content": html,
"html_file_path": str(archive_path) if archive_path else None,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"reference_favorite_id": reference_favorite_id,
"style_summary": style_summary,
"layout_summary": layout_summary,
}
)
except Exception:
logger.warning("Failed to persist scheduled_html_pages row for run %s", run_id, exc_info=True)
task_tz = ZoneInfo(task.get("timezone") or "UTC")
local_time = scheduled_for.astimezone(task_tz)
# ``scheduled_task_runs.result_json`` is TEXT (64 KB) on MySQL, so only
# inline the HTML when it comfortably fits. The full document always
# lives in ``scheduled_html_pages.html_content`` (LONGTEXT) + the
# archived ``index.html``; the ``/runs/{id}/html`` endpoints read from
# there, so previews work even when the inline copy is omitted here.
inline_html = html if len(html) <= _MAX_INLINE_RESULT_HTML else None
return {
"title": f"{title} - {local_time.strftime('%Y-%m-%d %H:%M')}",
"content": page_prompt or f"HTML 页面已生成:{title}",
"format": "html",
"html": inline_html,
"style_summary": style_summary,
"layout_summary": layout_summary,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"files": files,
"created_at": datetime.now(UTC).isoformat(),
}
async def _run_agent_attempt(
self,
task: dict[str, Any],
@@ -1275,3 +1624,132 @@ class ScheduledTaskService:
async def unsubscribe_task(self, user_id: str, task_id: str) -> bool:
return await self.store.delete_subscription(task_id, user_id)
# --- HTML pages / favorites ---
@property
def _html_store(self):
state = getattr(self.app, "state", None)
return getattr(state, "html_page_store", None)
async def get_html_page_for_run(self, run_id: str) -> dict[str, Any] | None:
"""Return the generated HTML page for a run (page-store row or run result)."""
store = self._html_store
if store is not None:
page = await store.get_page_by_run(run_id)
if page is not None:
return page
# Fallback to the inline HTML on the run result.
run = await self.store.get_run_any(run_id)
if run is None:
return None
result = run.get("result") if isinstance(run.get("result"), dict) else None
if not result or not isinstance(result.get("html"), str):
return None
return {
"run_id": run_id,
"task_id": run.get("task_id"),
"title": result.get("title"),
"html_content": result.get("html"),
"style_summary": result.get("style_summary"),
"layout_summary": result.get("layout_summary"),
"created_at": result.get("created_at"),
}
async def get_latest_html_page_for_task(self, task_id: str) -> dict[str, Any] | None:
store = self._html_store
if store is not None:
page = await store.latest_page_for_task(task_id)
if page is not None:
return page
# Fallback: newest run that carries inline HTML.
runs = await self.store.list_runs_any(task_id, limit=20)
for run in runs:
result = run.get("result") if isinstance(run.get("result"), dict) else None
if result and isinstance(result.get("html"), str):
return await self.get_html_page_for_run(run["id"])
return None
async def list_html_page_favorites(self, user_id: str) -> list[dict[str, Any]]:
store = self._html_store
if store is None:
return []
return await store.list_favorites(user_id)
async def get_html_page_favorite(self, user_id: str, favorite_id: str) -> dict[str, Any] | None:
store = self._html_store
if store is None:
return None
favorite = await store.get_favorite(favorite_id)
if favorite is None or favorite.get("user_id") != user_id:
return None
return favorite
async def delete_html_page_favorite(self, user_id: str, favorite_id: str) -> bool:
store = self._html_store
if store is None:
return False
return await store.delete_favorite(favorite_id, user_id)
async def create_html_page_favorite(
self,
user_id: str,
page_id: str,
*,
title: str | None = None,
description: str | None = None,
tags: list[str] | None = None,
) -> dict[str, Any] | None:
"""Bookmark a generated page, extracting a reusable style summary via GLM5.
Returns ``None`` when the page does not exist or is not owned by the
caller. Style extraction is best-effort — a failure still produces a
favorite (without the reuse hints) so the user's bookmark is never lost.
"""
store = self._html_store
if store is None:
return None
page = await store.get_page(page_id)
if page is None or page.get("user_id") != user_id:
return None
# A page can only be bookmarked once per user.
existing = await store.get_favorite_by_page(user_id, page_id)
if existing is not None:
raise DuplicateFavoriteError(page_id)
html = page.get("html_content") or ""
style_summary = page.get("style_summary")
layout_summary = page.get("layout_summary")
content_structure_summary = None
reuse_prompt = None
if html:
try:
# Reuse the model the page was generated with (recorded in the
# page's prompt snapshot) so favoriting doesn't fall back to the
# default model.
extract_model = (page.get("prompt_snapshot") or {}).get("model_name")
extract_prompt = html_page_utils.build_style_extract_prompt(html=html)
raw = await self._invoke_chat_model(extract_model, extract_prompt)
extracted = html_page_utils.extract_json_object(raw) or {}
style_summary = extracted.get("style_summary") or style_summary
layout_summary = extracted.get("layout_summary") or layout_summary
content_structure_summary = extracted.get("content_structure_summary")
reuse_prompt = extracted.get("reuse_prompt")
except Exception:
logger.warning("HTML style extraction failed for page %s; saving favorite without hints", page_id, exc_info=True)
return await store.create_favorite(
{
"id": str(uuid.uuid4()),
"user_id": user_id,
"page_id": page_id,
"title": (title or page.get("title") or "收藏页面").strip(),
"description": (description or None),
"tags": tags or [],
"html_snapshot": html,
"style_summary": style_summary,
"layout_summary": layout_summary,
"content_structure_summary": content_structure_summary,
"reuse_prompt": reuse_prompt,
}
)