feat(scheduled-tasks): 新增 HTML 页面定时任务(生成/预览/收藏/技能)
后端: - 新增 task_kind=html_page 任务类型,与普通定时任务并存,配置存 execution_context(无需迁移) - 执行流水线:智能体检索数据 -> 选定模型渲染 HTML -> 清洗/校验 -> 失败自动修复 -> 完整性闸门(缺 doctype/html/head/body/title 或残留脚本即失败) -> 归档 index.html -> 落库 - 新增 scheduled_html_pages / html_page_favorites 表与 store;收藏去重(不可重复)、可删除、收藏时提取可复用风格 - HTML 预览接口(鉴权 + 公开)、收藏接口 - 渲染注入已启用的 HTML 技能(metadata.kind=html_page);全程使用用户所选模型,去除硬编码 GLM-5 - 定时任务跳过 TitleMiddleware,避免标题生成回退默认模型 前端: - 创建/编辑弹窗增加任务类型与 HTML 配置(页面要求/限制/参考收藏页/模型下拉) - 运行详情页与公开页 iframe 沙箱预览 HTML;收藏按钮(已收藏禁用) - 参考收藏页可选可删 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
e1e9be8daa
commit
d9afaca51f
22 files changed
+1996
-107
No files matched your search
+365
@@ -0,0 +1,365 @@
|
||||
"""HTML page generation helpers for ``html_page`` scheduled tasks.
|
||||
|
||||
Pure, dependency-light utilities the scheduler uses to turn a GLM5 response
|
||||
into a safe, previewable HTML document:
|
||||
|
||||
- :func:`extract_json_object` — fault-tolerant JSON extraction from a model
|
||||
reply that may be wrapped in prose or ```json fences.
|
||||
- :func:`sanitize_html` — strip script-execution vectors (``<script>``,
|
||||
``<iframe>``, ``on*`` handlers, ``javascript:`` URLs, …) using BeautifulSoup.
|
||||
- :func:`validate_html` — cheap structural checks (doctype/html/head/body,
|
||||
non-empty body, no residual scripts) that gate the optional repair pass.
|
||||
- ``build_*_prompt`` — the GLM5 prompt templates (generation / repair / style
|
||||
extraction).
|
||||
- :func:`screenshot_html` — best-effort Playwright screenshot, a no-op when
|
||||
Playwright is not installed (the offline image ships without a browser).
|
||||
|
||||
Kept free of any ``app.*`` import so it stays inside the harness boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tags removed wholesale — they either execute code or load remote active
|
||||
# content that the iframe sandbox alone shouldn't be relied on to contain.
|
||||
_DANGEROUS_TAGS = frozenset(
|
||||
{"script", "iframe", "object", "embed", "applet", "frame", "frameset", "base"}
|
||||
)
|
||||
|
||||
# URL schemes that can execute script when navigated/loaded.
|
||||
_DANGEROUS_URL_RE = re.compile(r"^\s*(javascript|vbscript|data:text/html)", re.IGNORECASE)
|
||||
|
||||
# Attributes carrying a URL we must scheme-check.
|
||||
_URL_ATTRS = frozenset({"href", "src", "action", "formaction", "xlink:href", "background", "poster"})
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
|
||||
"""Split a ``---`` YAML frontmatter block from the body of a SKILL.md.
|
||||
|
||||
Returns ``(frontmatter_dict, body)``. When there is no frontmatter the dict
|
||||
is empty and the body is the whole text.
|
||||
"""
|
||||
if not text or not text.lstrip().startswith("---"):
|
||||
return {}, text or ""
|
||||
lines = text.splitlines()
|
||||
# Skip leading blank lines before the opening fence.
|
||||
start = 0
|
||||
while start < len(lines) and not lines[start].strip():
|
||||
start += 1
|
||||
end = None
|
||||
for i in range(start + 1, len(lines)):
|
||||
if lines[i].strip() == "---":
|
||||
end = i
|
||||
break
|
||||
if end is None:
|
||||
return {}, text
|
||||
fm_block = "\n".join(lines[start + 1 : end])
|
||||
body = "\n".join(lines[end + 1 :])
|
||||
try:
|
||||
import yaml
|
||||
|
||||
data = yaml.safe_load(fm_block) or {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
except Exception:
|
||||
data = {}
|
||||
return data, body
|
||||
|
||||
|
||||
def extract_json_object(text: str | None) -> dict[str, Any] | None:
|
||||
"""Best-effort parse of a JSON object from a model reply.
|
||||
|
||||
Handles three common shapes: a bare JSON object, a ```json fenced block,
|
||||
and a JSON object embedded in surrounding prose. Returns ``None`` when no
|
||||
balanced object can be recovered.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return None
|
||||
candidate = text.strip()
|
||||
|
||||
# Strip a single ```json ... ``` (or ``` ... ```) fence if present.
|
||||
fence = re.search(r"```(?:json)?\s*(.*?)```", candidate, flags=re.DOTALL | re.IGNORECASE)
|
||||
if fence:
|
||||
candidate = fence.group(1).strip()
|
||||
|
||||
# Fast path: the whole thing is already valid JSON.
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Fallback: scan for the first balanced ``{ ... }`` and parse that.
|
||||
start = candidate.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for index in range(start, len(candidate)):
|
||||
char = candidate[index]
|
||||
if in_string:
|
||||
if escape:
|
||||
escape = False
|
||||
elif char == "\\":
|
||||
escape = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
blob = candidate[start : index + 1]
|
||||
try:
|
||||
parsed = json.loads(blob)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
return None
|
||||
|
||||
|
||||
def sanitize_html(html: str) -> tuple[str, list[str]]:
|
||||
"""Remove script-execution vectors from ``html``.
|
||||
|
||||
Returns ``(cleaned_html, removed)`` where ``removed`` is a human-readable
|
||||
report of what was stripped (used for logging / repair feedback). The
|
||||
cleaner is defensive but not a full HTML validator — pair it with an
|
||||
``iframe sandbox`` (no ``allow-scripts``) on the preview side.
|
||||
"""
|
||||
if not html or not isinstance(html, str):
|
||||
return "", []
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except Exception: # pragma: no cover - bs4 is a declared dependency
|
||||
logger.warning("BeautifulSoup unavailable; returning HTML unsanitized")
|
||||
return html, []
|
||||
|
||||
removed: list[str] = []
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# 1) Drop dangerous tags entirely.
|
||||
for tag in soup.find_all(True):
|
||||
name = (tag.name or "").lower()
|
||||
if name in _DANGEROUS_TAGS:
|
||||
removed.append(f"<{name}>")
|
||||
tag.decompose()
|
||||
continue
|
||||
# External stylesheet / preload links pull remote resources.
|
||||
if name == "link":
|
||||
href = str(tag.get("href") or "")
|
||||
rel = " ".join(tag.get("rel") or []) if isinstance(tag.get("rel"), list) else str(tag.get("rel") or "")
|
||||
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
|
||||
removed.append("<link stylesheet>")
|
||||
tag.decompose()
|
||||
continue
|
||||
# meta refresh can redirect to arbitrary URLs.
|
||||
if name == "meta" and str(tag.get("http-equiv") or "").lower() == "refresh":
|
||||
removed.append("<meta refresh>")
|
||||
tag.decompose()
|
||||
continue
|
||||
|
||||
# 2) Strip event handlers and dangerous URL schemes from surviving tags.
|
||||
for tag in soup.find_all(True):
|
||||
for attr in list(tag.attrs.keys()):
|
||||
lowered = attr.lower()
|
||||
if lowered.startswith("on"):
|
||||
removed.append(f"{lowered}=")
|
||||
del tag.attrs[attr]
|
||||
continue
|
||||
if lowered in _URL_ATTRS:
|
||||
value = tag.attrs[attr]
|
||||
value_str = value if isinstance(value, str) else " ".join(value)
|
||||
if _DANGEROUS_URL_RE.match(value_str or ""):
|
||||
removed.append(f"{lowered}={value_str[:24]}")
|
||||
del tag.attrs[attr]
|
||||
|
||||
return str(soup), removed
|
||||
|
||||
|
||||
def validate_html(html: str) -> list[str]:
|
||||
"""Return a list of structural problems with ``html`` (empty == valid).
|
||||
|
||||
A page only counts as a complete document when it has the full skeleton
|
||||
(doctype/html/head/body), a non-empty ``<title>``, visible body content
|
||||
(no white-screen), no residual scripts, and no external stylesheet links.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
if not html or not isinstance(html, str):
|
||||
return ["empty html"]
|
||||
lowered = html.lower()
|
||||
if "<!doctype html" not in lowered:
|
||||
errors.append("缺少 <!doctype html> 声明")
|
||||
if "<html" not in lowered:
|
||||
errors.append("缺少 <html> 根元素")
|
||||
if "<head" not in lowered:
|
||||
errors.append("缺少 <head> 元素")
|
||||
if "<body" not in lowered:
|
||||
errors.append("缺少 <body> 元素")
|
||||
if "<script" in lowered:
|
||||
errors.append("仍包含 <script> 标签(不允许脚本)")
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
# Title must exist and be non-empty.
|
||||
title_el = soup.find("title")
|
||||
if title_el is None or not title_el.get_text(strip=True):
|
||||
errors.append("缺少非空 <title> 标题")
|
||||
# External stylesheet links pull remote resources — not allowed offline.
|
||||
for link in soup.find_all("link"):
|
||||
href = str(link.get("href") or "")
|
||||
rel = " ".join(link.get("rel") or []) if isinstance(link.get("rel"), list) else str(link.get("rel") or "")
|
||||
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
|
||||
errors.append("包含外链样式表(不允许外部 CDN/资源)")
|
||||
break
|
||||
# Body must carry some visible content — guards against white-screen.
|
||||
body = soup.body or soup
|
||||
text = body.get_text(strip=True) if body else ""
|
||||
has_media = bool(body.find(["img", "svg", "canvas", "table", "ul", "ol"])) if body else False
|
||||
if len(text) < 5 and not has_media:
|
||||
errors.append("页面正文为空,可能白屏")
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt templates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_GENERATION_TEMPLATE = """你是一个内网页面生成助手。请根据输入数据生成一个完整的 HTML 页面。
|
||||
|
||||
用户页面要求:
|
||||
{page_prompt}
|
||||
|
||||
限制条件:
|
||||
{negative_prompt}
|
||||
|
||||
本次数据:
|
||||
{data}
|
||||
|
||||
参考页面风格:
|
||||
{reference_style}
|
||||
|
||||
参考页面布局:
|
||||
{reference_layout}
|
||||
{skill_instructions}
|
||||
要求:
|
||||
1. 只输出 JSON,不要输出 Markdown,不要输出多余解释。
|
||||
2. JSON 字段必须包含 title、html、style_summary、layout_summary。
|
||||
3. html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
|
||||
4. 所有 CSS 写在 <style> 标签内,不要引用外部 CDN 或外链样式表。
|
||||
5. 不要包含 <script>、外链脚本、iframe、表单提交、on* 事件属性。
|
||||
6. 参考页面只用于风格和布局,页面内容必须来自“本次数据”,不要复用参考页旧数据。
|
||||
7. 页面使用中文,排版适合在浏览器中直接预览。
|
||||
"""
|
||||
|
||||
_REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问题,请修复后重新输出完整页面。
|
||||
|
||||
原始页面要求:
|
||||
{requirements}
|
||||
|
||||
发现的问题:
|
||||
{errors}
|
||||
|
||||
待修复 HTML:
|
||||
{html}
|
||||
|
||||
要求:
|
||||
1. 只输出 JSON,字段包含 fixed_html、changes(修复说明数组)。
|
||||
2. fixed_html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
|
||||
3. 移除所有 <script>、iframe、外链脚本、on* 事件属性、表单提交。
|
||||
4. 保留原有内容与数据,只修复结构与安全问题。
|
||||
"""
|
||||
|
||||
_STYLE_EXTRACT_TEMPLATE = """你是一个页面风格分析助手。请分析下面的 HTML 页面,提取可复用的风格与布局特征。
|
||||
|
||||
HTML 页面:
|
||||
{html}
|
||||
|
||||
要求:
|
||||
1. 只输出 JSON,字段包含 style_summary、layout_summary、content_structure_summary、reuse_prompt。
|
||||
2. style_summary:颜色、字体、间距、圆角、阴影、视觉密度等视觉风格。
|
||||
3. layout_summary:页面分区、卡片、表格、图表、导航等布局结构。
|
||||
4. content_structure_summary:信息组织顺序与层级。
|
||||
5. reuse_prompt:一段可直接拼接到下次生成 prompt 的中文风格说明,不要包含本页的具体数据。
|
||||
"""
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int) -> str:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||||
text = text or ""
|
||||
return text if len(text) <= limit else text[:limit] + "\n…(内容过长已截断)"
|
||||
|
||||
|
||||
def build_generation_prompt(
|
||||
*,
|
||||
page_prompt: str,
|
||||
negative_prompt: str | None,
|
||||
data: Any,
|
||||
reference_style: str | None,
|
||||
reference_layout: str | None,
|
||||
skill_instructions: str | None = None,
|
||||
data_limit: int = 16000,
|
||||
skill_limit: int = 16000,
|
||||
) -> str:
|
||||
skill_block = ""
|
||||
if skill_instructions and skill_instructions.strip():
|
||||
skill_block = "\n可用页面生成技能规范(请严格遵循):\n" + _clip(skill_instructions.strip(), skill_limit) + "\n"
|
||||
return _GENERATION_TEMPLATE.format(
|
||||
page_prompt=(page_prompt or "(未提供,使用通用看板风格)").strip(),
|
||||
negative_prompt=(negative_prompt or "无").strip(),
|
||||
data=_clip(data, data_limit),
|
||||
reference_style=(reference_style or "无").strip(),
|
||||
reference_layout=(reference_layout or "无").strip(),
|
||||
skill_instructions=skill_block,
|
||||
)
|
||||
|
||||
|
||||
def build_repair_prompt(*, html: str, errors: list[str], requirements: str | None) -> str:
|
||||
return _REPAIR_TEMPLATE.format(
|
||||
requirements=(requirements or "无").strip(),
|
||||
errors="\n".join(f"- {e}" for e in errors) or "- 未知问题",
|
||||
html=_clip(html, 32000),
|
||||
)
|
||||
|
||||
|
||||
def build_style_extract_prompt(*, html: str) -> str:
|
||||
return _STYLE_EXTRACT_TEMPLATE.format(html=_clip(html, 32000))
|
||||
|
||||
|
||||
async def screenshot_html(html: str, output_path) -> str | None:
|
||||
"""Best-effort PNG screenshot of ``html`` for white-screen / console QA.
|
||||
|
||||
Returns the screenshot path on success, or ``None`` when Playwright (or a
|
||||
usable browser) is unavailable — the offline image ships without one, so
|
||||
this must degrade silently rather than fail the run.
|
||||
"""
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch()
|
||||
try:
|
||||
page = await browser.new_page()
|
||||
await page.set_content(html, wait_until="networkidle")
|
||||
await page.screenshot(path=str(output_path), full_page=True)
|
||||
finally:
|
||||
await browser.close()
|
||||
return str(output_path)
|
||||
except Exception:
|
||||
logger.debug("Playwright screenshot failed; skipping QA screenshot", exc_info=True)
|
||||
return None
|
||||
Reference in new issue
Block a user