This commit is contained in:
许海洋 committed 2026-06-04 08:12:01 +08:00
commit 3e2182acc5
275 files changed
+29592 -11370

No files matched your search

@@ -74,6 +74,28 @@ def get_model(config: RunnableConfig, model_name: str | None = None):
return create_chat_model(name=name)
def get_writing_model(config: RunnableConfig, state: dict):
"""大纲 / 写作 / 审核专用:严格使用用户在表单选定的模型,**不回退默认**。
用户模型在 intent_parser(首次 invocation,configurable.model_name 可用)阶段
捕获进 ``state['model_name']``,经 checkpoint 持久化。这三个节点都在
「确认素材 / 大纲」resume 之后才运行,而 resume 是新 invocation、其 config 不再
带 model_name —— 所以必须从 state 读,不能从 configurable 读(否则回退到 config
列表第一个模型,常是推理模型,既影响质量也会让 reasoning_content 走不出流式)。
缺失视为配置错误直接抛 ValueError —— 由各节点的 try/except 兜底成 status:error,
而不是静默回退默认模型。
"""
name = ((state or {}).get("model_name") or "").strip()
if not name:
# 兜底再看一眼 configurable(首次同 invocation 内直跑到这里的极端情况)
name = (config.get("configurable", {}).get("model_name", "") or "").strip()
if not name:
raise ValueError("未指定写作模型(model_name 为空):请在写作表单选择模型后重试")
from deerflow.models import create_chat_model
return create_chat_model(name=name)
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
_FULLWIDTH_FIX = {
"“": '"', # 左双引号 “
@@ -175,6 +197,21 @@ def parse_json(text: str, fallback: dict) -> dict:
return fallback
def reasoning_delta(chunk) -> str:
"""提取流式 chunk 的增量「推理内容」。
推理模型(deepseek-reasoner / GLM 等)会把思考过程放在
``additional_kwargs.reasoning_content`` 增量字段里,而 ``content`` 为空 ——
只读 content 的流式循环会把这些 chunk 全部跳过,前端就看不到任何实时输出。
各写作节点用本函数把推理增量当作 ``is_thinking=True`` 推给前端的思考框。
兼容不同 provider 的字段名:``reasoning_content``(主流)/ ``reasoning``。
"""
ak = getattr(chunk, "additional_kwargs", None) or {}
val = ak.get("reasoning_content") or ak.get("reasoning") or ""
return val if isinstance(val, str) else ""
def _content_to_text(content) -> str:
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
if isinstance(content, str):
@@ -11,7 +11,7 @@ from ..state import AIWritingState, ReviewResult, ReviewIssue
from ..prompts.editor_prompts import (
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
)
from ._utils import get_ai_writing_config, get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
from ._utils import get_ai_writing_config, get_writing_model, parse_json, _content_to_text, reasoning_delta, ThinkTagParser, safe_get_stream_writer
logger = logging.getLogger(__name__)
@@ -49,6 +49,10 @@ async def _stream_one_review(
SystemMessage(content=EDITOR_SYSTEM),
HumanMessage(content=user_prompt),
]):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
writer({"type": "review_chunk", "category": category, "text": reasoning, "is_thinking": True})
delta = _content_to_text(chunk.content)
if not delta:
continue
@@ -118,7 +122,7 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
model = get_writing_model(config, state)
except Exception as e:
logger.error("editor model creation failed: %s", e, exc_info=True)
return {
@@ -14,6 +14,7 @@ from ._utils import (
_content_to_text,
get_model,
parse_json,
reasoning_delta,
safe_get_stream_writer,
)
@@ -215,6 +216,10 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
async for chunk in model.astream(
[SystemMessage(content=system), HumanMessage(content=user)]
):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
writer({"type": "intent_chunk", "text": reasoning, "is_thinking": True})
delta = _content_to_text(chunk.content)
if not delta:
continue
@@ -259,6 +264,12 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
updates: dict = {"status": "researching", "progress_events": progress_events}
# 捕获用户表单选定的主模型进 state(仅本节点能可靠拿到 configurable.model_name —
# 它运行在首次 invocation;后续大纲/写作/审核在 resume 后跑,config 已不带它)。
# 大纲/写作/审核经 get_writing_model 严格读 state['model_name'],不回退默认。
selected_model = (configurable.get("model_name") or "").strip()
if selected_model:
updates["model_name"] = selected_model
updates["article_type"] = final_article_type
updates["target_audience"] = final_target_audience
updates["word_count_target"] = final_word_count
@@ -1,9 +1,11 @@
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
"""素材收集专家节点:关键词提取 → 并发搜索(含笔记) → 相关度排序。"""
from __future__ import annotations
import asyncio
import logging
import os
import re
import time
import uuid
from typing import List
@@ -20,6 +22,10 @@ from ._utils import get_ai_writing_config, llm_json, safe_get_stream_writer
logger = logging.getLogger(__name__)
_NOTEBOOK_SEARCH_PATH = "/api/v1/source/mentioned-documents/search"
_NOTEBOOK_MAX_KEYWORDS = 3 # 每次写作最多对笔记发起的检索轮数
_NOTEBOOK_TOP_K = 5 # 每次检索取回的 chunk 数
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 50
_DEFAULT_MAX_MATERIALS = 50
_MIN_MAX_MATERIALS = 1
@@ -29,6 +35,11 @@ _MAX_MAX_MATERIALS = 100
_DEFAULT_KEYWORD_COUNT = 6
_MAX_KEYWORD_COUNT = 10
# 单个技能的 web 检索超时(秒)。技能检索是顺序循环、最多 5 个技能,
# DuckDuckGo 等 provider 偶发限速/挂起会把整段"技能检索过程"拖到几十秒看起来卡死;
# 超时即放弃该技能、emit 0 条结果继续下一个,保证时间线持续推进。
_SKILL_SEARCH_TIMEOUT_SECONDS = 25
def _resolve_keyword_count(raw: object) -> int:
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
@@ -89,6 +100,67 @@ def _notice(title: str, detail: str, raw: str = "") -> dict:
return event
async def _search_notebook(
keywords: list[str],
search_space_id: int,
source_ids: list[int],
token: str,
) -> list[Material]:
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
import httpx # 懒导入:仅在笔记模式下才需要
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
if not base:
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
return []
results: list[Material] = []
mat_idx = 0
try:
async with httpx.AsyncClient(timeout=20) as client:
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
try:
resp = await client.post(
f"{base}{_NOTEBOOK_SEARCH_PATH}",
headers={"Authorization": f"Bearer {token}"},
json={
"query": kw,
"source_ids": source_ids,
"search_space_id": search_space_id,
"top_k": _NOTEBOOK_TOP_K,
},
)
resp.raise_for_status()
data = resp.json()
# 解析 documents 列表,每个 chunk 作为独立素材
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
for doc in documents or []:
doc_title: str = doc.get("title") or "笔记文档"
doc_url: str = doc.get("url") or ""
for chunk in doc.get("chunks") or []:
chunk_content: str = (chunk.get("content") or "").strip()
if not chunk_content:
continue
mat_idx += 1
raw_score = chunk.get("score")
chunk_score = float(raw_score) if raw_score is not None else max(0.5, 0.95 - (mat_idx - 1) * 0.03)
results.append(Material(
id=f"nb{mat_idx:03d}",
title=doc_title,
content=chunk_content[:500],
source=f"笔记·{doc_title}",
source_type="notebook",
relevance_score=chunk_score,
url=doc_url,
))
except Exception as e:
logger.warning("notebook search kw=%r failed: %s", kw, e)
except Exception as e:
logger.warning("notebook search client error: %s", e)
return results
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
intent = state["user_intent"]
article_type = state.get("article_type", "analysis")
@@ -110,11 +182,19 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
is_user_query_search = action == "re_search" and bool(user_query)
# 兼容旧版"追加关键词"调用方(如果还有 client 传 extra_keywords 数组而非 user_query)
is_append_only = action == "re_search" and not user_query and bool(extra_keywords)
# PAUSE-1 素材确认卡用户点「启用技能检索」:
# 用户在写作表单选择的素材来源:knowledge_base(默认 web/intranet 检索)/ skill(技能检索)。
# 首次检索(没有用户干预)时按它决定检索方式;缺省 knowledge_base。
material_source = (state.get("material_source") or "knowledge_base").strip()
is_first_run = not intervention
is_initial_skill_source = is_first_run and material_source == "skill"
# 技能检索触发条件(任一即可):
# - 首次检索且表单选了「技能检索」来源
# - PAUSE-1 素材确认卡用户点「启用技能检索」按钮
# 技能检索流程:
# - 不再走 web/intranet 检索,改用 SkillSearchProvider
# - 顺序调用候选技能 + 早停(第一个技能够就停,不够再加下一个)
# - 旧 material_package 完全丢弃(覆盖式替换,让用户对比 web vs skill 结果)
is_skill_search = action == "enable_skill_search"
is_skill_search = action == "enable_skill_search" or is_initial_skill_source
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
@@ -167,6 +247,22 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
notebook_materials: list[Material] = []
if notebook_search_space_id:
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
notebook_token: str = cfg.get("notebook_token") or ""
_emit("正在检索笔记内容…")
notebook_materials = await _search_notebook(
keywords[:_NOTEBOOK_MAX_KEYWORDS],
notebook_search_space_id,
notebook_source_ids,
notebook_token,
)
if notebook_materials:
_emit(f"从笔记中获取到 {len(notebook_materials)} 条素材")
# Step 2: 搜索
# 技能检索:技能不是素材,是"检索策略 / 视角"。流程:
# a. SkillPicker 选出按相关度排序的 skills(推 list_skills + select_skills 事件)
@@ -178,6 +274,10 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
def _emit_skill_step(step: dict) -> None:
skill_steps.append(step)
# 技能检索时机 trace:grep "[skill-timing]" 看每步 emit 的真实时刻,
# 与前端 [ai-writing:custom] 日志对齐,钉死是否实时推送。
logger.info("[skill-timing] t=%.3f emit skill_search_step type=%s skill=%s",
time.monotonic(), step.get("type"), step.get("skill_name") or "")
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
if is_skill_search:
@@ -205,15 +305,25 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
for entry in selected:
sk = entry["skill"]
# 用 LLM 把 user_query + skill.description 转成针对性关键词
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
# invoke_skill 步骤带上实际检索词(截断防过长),让用户在每次长 await 之前
# 就看到"这一步在用技能 X 检索什么",配合前端 spinner 体现进行中。
_emit_skill_step({
"type": "invoke_skill",
"message": f"使用技能「{sk.name}」开始检索…",
"message": f"使用技能「{sk.name}」检索:{skill_query[:60]}",
"skill_name": sk.name,
})
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
try:
res = await web_provider.search(skill_query, max_results=per_skill)
# 加超时兜底:单个技能检索最多等 _SKILL_SEARCH_TIMEOUT_SECONDS 秒,
# 超时不再阻塞后续技能,避免整段过程看起来"冻结"。
res = await asyncio.wait_for(
web_provider.search(skill_query, max_results=per_skill),
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError:
logger.warning("skill[%s] search timed out after %ss", sk.name, _SKILL_SEARCH_TIMEOUT_SECONDS)
search_errors.append(f"skill[{sk.name}]: 检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)")
res = []
except Exception as e:
logger.warning("skill[%s] search failed: %s", sk.name, e)
search_errors.append(f"skill[{sk.name}]: {e}")
@@ -302,10 +412,17 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
existing_pkg = state.get("material_package")
if is_skill_search:
# 技能编排循环已经完成素材累积,直接用 raw_materials_pre 当 raw_materials
# 笔记素材追加在技能素材之后
raw_materials: list[Material] = raw_materials_pre # type: ignore[name-defined]
raw_materials.extend(notebook_materials)
else:
seen_urls: set[str] = set()
raw_materials = []
# 笔记素材优先排在最前面,赋予高相关度
raw_materials = list(notebook_materials)
for m in notebook_materials:
if m.get("url"):
seen_urls.add(m["url"])
preserve_existing = existing_pkg and (is_supplement or is_append_only)
if preserve_existing:
for m in existing_pkg.get("materials", []):
@@ -12,7 +12,7 @@ from ..prompts.writer_prompts import (
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
)
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser, safe_get_stream_writer
from ._utils import get_writing_model, _content_to_text, count_words, reasoning_delta, ThinkTagParser, safe_get_stream_writer
logger = logging.getLogger(__name__)
@@ -197,7 +197,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
model = get_writing_model(config, state)
except Exception as e:
logger.error("draft model creation failed: %s", e, exc_info=True)
return {
@@ -250,6 +250,17 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
SystemMessage(content=DRAFT_SECTION_SYSTEM),
HumanMessage(content=user_msg),
]):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
writer({
"type": "draft_chunk",
"section_index": i,
"section_title": sec_title,
"text": reasoning,
"is_thinking": True,
"is_section_done": False,
})
delta = _content_to_text(chunk.content)
if not delta:
continue
@@ -7,11 +7,19 @@ import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, KeyPoint, Outline, OutlineSection
from ..outline_parser import parse_user_outline
from ..prompts.writer_prompts import (
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
OUTLINE_MATERIAL_ASSIGN_SYSTEM,
OUTLINE_MATERIAL_ASSIGN_USER,
OUTLINE_NORMALIZE_SYSTEM,
OUTLINE_NORMALIZE_USER,
OUTLINE_REVISE_SYSTEM,
OUTLINE_REVISE_USER,
OUTLINE_SYSTEM,
OUTLINE_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
from ..state import AIWritingState, KeyPoint, Outline, OutlineSection
from ._utils import ThinkTagParser, _content_to_text, get_writing_model, llm_json, parse_json, reasoning_delta, safe_get_stream_writer
logger = logging.getLogger(__name__)
@@ -80,6 +88,108 @@ def _parse_outline(raw: dict) -> Outline:
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
def _derive_title(state: AIWritingState) -> str:
"""用户大纲没给总标题时,用写作意图兜底一个标题。"""
intent = (state.get("user_intent") or "").strip()
if not intent:
return "未命名文章"
return intent[:30].rstrip() + ("…" if len(intent) > 30 else "")
async def _normalize_user_outline_via_llm(state: AIWritingState, config: RunnableConfig, raw: str) -> Outline:
"""确定性解析失败时,调 LLM 把用户大纲原文归一成三级结构(严格保真、不改写)。
再失败则把每个非空行当作一个章节,保证永远产出一份可用大纲、不丢用户内容。
"""
model_name = (state.get("model_name") or "").strip() or None
line_fallback = {
"title": "",
"sections": [
{"section_title": ln.strip(), "key_points": [], "material_ids": []}
for ln in raw.splitlines() if ln.strip()
] or [{"section_title": "正文", "key_points": [], "material_ids": []}],
}
result = await llm_json(
OUTLINE_NORMALIZE_SYSTEM,
OUTLINE_NORMALIZE_USER.format(raw_outline=raw),
line_fallback,
config,
model_name=model_name,
)
return _parse_outline(result)
async def _assign_materials_to_outline(
state: AIWritingState, config: RunnableConfig, outline: Outline, pkg: dict
) -> Outline:
"""为用户大纲的各章节回填 material_ids(结构锁定:只取编号、按章节序号嫁接)。
LLM 只负责「哪个章节配哪些素材」,标题与层级一律以解析结果为准,杜绝改写。失败则
保持 material_ids 为空(writer_draft 自带「取前 2 条素材」兜底)。
"""
sections = outline.get("sections", [])
materials = pkg.get("materials") or []
if not sections or not materials:
return outline
valid_ids = {m["id"] for m in materials}
sections_list = "\n".join(
f"{i + 1}. {s.get('section_title', '')}" for i, s in enumerate(sections)
)
model_name = (state.get("model_name") or "").strip() or None
result = await llm_json(
OUTLINE_MATERIAL_ASSIGN_SYSTEM,
OUTLINE_MATERIAL_ASSIGN_USER.format(
user_intent=state.get("user_intent", ""),
sections_list=sections_list,
materials_summary=_materials_summary(pkg),
),
{"assignments": []},
config,
model_name=model_name,
)
assignments = result.get("assignments")
if not isinstance(assignments, list):
return outline
for i, section in enumerate(sections):
if i >= len(assignments):
break
ids = assignments[i]
if isinstance(ids, list):
section["material_ids"] = [mid for mid in ids if mid in valid_ids]
return outline
async def _build_outline_from_user(
state: AIWritingState, config: RunnableConfig, pkg: dict, raw: str
) -> dict:
"""用户自带大纲分支:解析用户大纲 → 关联素材 → 直接进确认,不调 LLM 重新规划。"""
writer = safe_get_stream_writer()
writer({"type": "outline_chunk", "text": "正在整理你提供的大纲…\n", "is_thinking": True})
outline = parse_user_outline(raw)
if outline is None:
# 确定性解析拿不准(散文化/格式混乱)→ LLM 兜底归一
outline = await _normalize_user_outline_via_llm(state, config, raw)
if not outline.get("title"):
outline = {**outline, "title": _derive_title(state)}
# 仍走了检索 → 把已检索素材关联到用户章节(结构锁定)
outline = await _assign_materials_to_outline(state, config, outline, pkg)
section_count = len(outline.get("sections", []))
return {
"current_outline": outline,
"status": "awaiting_outline_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": "采用你提供的大纲"},
{"type": "outline_ready", "agent_name": "作家", "message": f"已整理你的大纲,共 {section_count} 个章节"},
],
}
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
@@ -95,6 +205,13 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
current_outline = state.get("current_outline")
is_revise = bool(feedback) and bool(current_outline)
# 用户自带大纲:首轮(尚无大纲、非重规划)且表单粘了大纲原文 → 走「采用用户大纲」分支,
# 跳过 LLM 重新规划。后续若用户在确认卡里点「重新规划」,current_outline 已存在 → 落到
# 下方 is_revise 编辑模式,按反馈调整用户大纲(符合预期)。
user_outline_raw = (state.get("user_outline_raw") or "").strip()
if user_outline_raw and not current_outline and not is_revise:
return await _build_outline_from_user(state, config, pkg, user_outline_raw)
if is_revise:
system_prompt = OUTLINE_REVISE_SYSTEM
user_prompt = OUTLINE_REVISE_USER.format(
@@ -125,13 +242,17 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
parser = ThinkTagParser()
try:
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
# get_writing_model 放进 try:用户模型缺失 / 模型名配错时抛 ValueError,
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
model = get_model(config)
model = get_writing_model(config, state)
async for chunk in model.astream([
SystemMessage(content=system_prompt),
HumanMessage(content=user_prompt),
]):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
writer({"type": "outline_chunk", "text": reasoning, "is_thinking": True})
delta = _content_to_text(chunk.content)
if not delta:
continue
@@ -0,0 +1,220 @@
"""把用户粘贴的大纲文本解析成三级 ``Outline`` 结构(确定性解析,不依赖 LLM)。
用户在写作表单里可能直接粘贴一份已有大纲,要求「完全按我的大纲生成」。本模块负责
把各种常见的大纲文本格式严格、保真地转成后端三级结构:
Outline.sections[] = Level 1 章节(section_title)
OutlineSection.key_points[] = Level 2 小节(point_title)
KeyPoint.sub_points[] = Level 3 子要点
兼容的输入格式:
- Markdown 标题:``#`` / ``##`` / ``###`` / ``####``(首个孤立 ``#`` 视为文章标题)
- 中文编号:``一、`` / ``(一)`` / ``第一章`` / ``第一节``
- 阿拉伯编号:``1.`` / ``1、`` / ``1.1`` / ``1.1.1``
- 缩进 bullet:``-`` / ``*`` / ``•`` / ``·``(用缩进深度表示层级)
- 纯文本平铺:每行一个一级标题(短标题、无句末标点)
设计原则:
- **保真不改写**:只识别层级、原样保留标题文本,绝不增删或改写用户内容。
- **多级混排自适应**:把文本里实际出现的各种「层级标记」收集起来排序,按相对外层
顺序映射到 1/2/3 级——所以不依赖固定的标记搭配,``##``+``-`` 或 ``一、``+缩进
都能正确分层。
- **拿不准就交给上层**:识别不出可靠层级(如整段散文)时返回 ``None``,由调用方走
LLM 兜底归一,避免硬把散文拆成错误结构。
"""
from __future__ import annotations
import re
from .state import KeyPoint, Outline, OutlineSection
# ── 行首标记识别 ──────────────────────────────────────────────────────────────
_CN_NUM = "一二三四五六七八九十百千零〇两"
# Markdown 标题:## 标题
_MD_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*\S)\s*$")
# 第一章 / 第1节 / 第一部分
_CN_CHAPTER_RE = re.compile(rf"^第\s*[{_CN_NUM}\d]+\s*[章节部分篇讲回卷](?:[\s::、..))]\s*|\s+)?(.*)$")
# 一、 / 十二、 / 一. / 一.
_CN_DUN_RE = re.compile(rf"^[{_CN_NUM}]+\s*[、..]\s*(.*)$")
# (一) / (一) / (1) / (1)
_PAREN_RE = re.compile(rf"^[((]\s*[{_CN_NUM}\d]+\s*[))]\s*(.*)$")
# 1.1 / 1.1.1(带至少一个点的多级阿拉伯编号)
_ARABIC_DOTTED_RE = re.compile(r"^(\d+(?:\.\d+)+)\s*[、..))]?\s*(.*)$")
# 1. / 1、 / 1) / 1.(单级阿拉伯编号)
_ARABIC_SINGLE_RE = re.compile(r"^\d+\s*[、..))]\s*(.*)$")
# 带圈数字 ①②③ / 罗马 ⅰ
_CIRCLED_RE = re.compile(r"^[①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮]\s*(.*)$")
# bullet:- / * / • / ·(要求标记后有空白,避免误吞「*强调*」之类)
_BULLET_RE = re.compile(r"^[-*•·]\s+(.*\S)\s*$")
# 标记分组的「外层度」基准值(越小越外层)。group: 0=标题 1=编号 2=缩进。
# 同组内 order 决定相对层级;不同组之间靠 group 排序(标题 < 编号 < 缩进)。
# 编号顺序遵循中文公文惯例:第一章 > 一、 > (一) > 1. > 1.1 > ①。
_NUM_ORDER = {
"cn_chapter": 0.0, # 第一章
"cn_dun": 1.0, # 一、
"paren": 2.0, # (一)
"arabic_single": 3.0, # 1.
"circled": 8.0, # ①
}
def _indent_width(line: str) -> int:
"""计算行首缩进宽度(tab 记 4 列)。"""
width = 0
for ch in line:
if ch == " ":
width += 1
elif ch == "\t":
width += 4
else:
break
return width
def _classify(stripped: str, indent: int) -> tuple[tuple[int, float], str] | None:
"""识别一行的层级标记,返回 ``((group, order), 纯文本)``;无法识别返回 None。
``(group, order)`` 是该行的「层级键」:把全文出现过的不同键排序后,按位置映射到
1/2/3 级。这样不必预设标记搭配,相对顺序对了层级就对。
"""
if not stripped:
return None
# 1) Markdown 标题(group 0,order = 井号数)
m = _MD_HEADING_RE.match(stripped)
if m:
depth = len(m.group(1))
text = m.group(2).strip()
return (0, float(depth)), text
# 2) 各类编号(group 1)
m = _CN_CHAPTER_RE.match(stripped)
if m and stripped.startswith("第"):
return (1, _NUM_ORDER["cn_chapter"]), (m.group(1) or "").strip()
m = _CN_DUN_RE.match(stripped)
if m:
return (1, _NUM_ORDER["cn_dun"]), m.group(1).strip()
m = _ARABIC_DOTTED_RE.match(stripped)
if m:
depth = m.group(1).count(".") + 1 # 1.1 → 2, 1.1.1 → 3
# 多级阿拉伯:order 随层级递增,落在单级(3.0)与圈号(8.0)之间
return (1, 3.0 + depth), m.group(2).strip()
m = _PAREN_RE.match(stripped)
if m:
return (1, _NUM_ORDER["paren"]), m.group(1).strip()
m = _ARABIC_SINGLE_RE.match(stripped)
if m:
return (1, _NUM_ORDER["arabic_single"]), m.group(1).strip()
m = _CIRCLED_RE.match(stripped)
if m:
return (1, _NUM_ORDER["circled"]), m.group(1).strip()
# 3) bullet:用缩进深度区分层级(group 2)
m = _BULLET_RE.match(stripped)
if m:
return (2, float(indent)), m.group(1).strip()
# 4) 纯文本:用缩进深度区分层级(group 3,最内层基准)
return (3, float(indent)), stripped
def _looks_like_prose(text: str) -> bool:
"""判断一行是否更像散文句子而非大纲标题(长、或带句末标点)。"""
if len(text) > 50:
return True
return text.rstrip()[-1:] in "。!?!?;;"
def parse_user_outline(raw: str) -> Outline | None:
"""把用户粘贴的大纲文本解析成三级 ``Outline``;无法可靠解析时返回 ``None``。
返回 ``None`` 的情形:空白输入、只有一行且无任何标记、或整体更像散文段落——这些交给
上层用 LLM 兜底归一,避免误拆。
"""
if not raw or not raw.strip():
return None
rows: list[tuple[tuple[int, float], str]] = []
for line in raw.splitlines():
if not line.strip():
continue
indent = _indent_width(line)
classified = _classify(line.strip(), indent)
if classified is None:
continue
key, text = classified
if text:
rows.append((key, text))
if not rows:
return None
marked = any(key[0] != 3 for key, _ in rows) # 是否出现过标题/编号/bullet 标记
# 纯文本且无任何标记:可能是「平铺标题列表」也可能是散文。
# 单行、或任一行像句子 → 判为散文,返回 None 交给 LLM。
if not marked:
if len(rows) < 2:
return None
if any(_looks_like_prose(text) for _, text in rows):
return None
# 文章标题:仅在「Markdown 首行是孤立的最浅井号、且存在更深井号」时认定。
title = ""
body = rows
heading_depths = [key[1] for key, _ in rows if key[0] == 0]
if heading_depths:
min_depth = min(heading_depths)
first_key = rows[0][0]
is_first_top_heading = first_key == (0, min_depth)
only_one_at_top = sum(1 for d in heading_depths if d == min_depth) == 1
has_deeper = any(d > min_depth for d in heading_depths)
if is_first_top_heading and only_one_at_top and has_deeper:
title = rows[0][1]
body = rows[1:]
if not body:
return None
# 把 body 里出现的不同「层级键」排序,按位置映射到 1/2/3 级(超过 3 级压到 3)。
distinct_keys = sorted({key for key, _ in body})
level_of: dict[tuple[int, float], int] = {
key: min(idx + 1, 3) for idx, key in enumerate(distinct_keys)
}
sections: list[OutlineSection] = []
cur_section: OutlineSection | None = None
cur_point: KeyPoint | None = None
for key, text in body:
level = level_of[key]
if level <= 1:
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
sections.append(cur_section)
cur_point = None
elif level == 2:
if cur_section is None: # 孤立 L2:提升为章节,避免空标题
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
sections.append(cur_section)
cur_point = None
continue
cur_point = KeyPoint(point_title=text, sub_points=[])
cur_section["key_points"].append(cur_point)
else: # level == 3
if cur_section is None: # 孤立 L3:提升为章节
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
sections.append(cur_section)
cur_point = None
elif cur_point is None: # 章节下尚无小节:提升为小节
cur_point = KeyPoint(point_title=text, sub_points=[])
cur_section["key_points"].append(cur_point)
else:
cur_point["sub_points"].append(text)
if not sections:
return None
return Outline(title=title, sections=sections)
@@ -123,6 +123,60 @@ OUTLINE_REVISE_USER = """写作意图:{user_intent}
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
# ── 用户自带大纲:LLM 兜底归一(确定性解析失败时才用)──────────────────────────
OUTLINE_NORMALIZE_SYSTEM = """你是一名大纲格式整理助手。
用户粘贴了一份已有的文章大纲(格式可能不规范),你的任务是把它原样转成结构化 JSON。
【最高原则:保真,绝不改写】
- 严格保留用户写的每一条标题文字,逐字照搬,不要润色、改写、翻译、增加或删除任何条目。
- 你只判断层级(哪条是章节、哪条是小节、哪条是子要点),不创作内容。
- 层级判断依据用户原文里的编号(一、/(一)/1./1.1)、Markdown 井号数、或缩进深度。
【三级结构】
- Level 1 章节 → section_title
- Level 2 小节 → key_points[].point_title
- Level 3 子要点 → key_points[].sub_points[](字符串数组)
- 用户大纲只有一级时,每条都是章节,key_points 留空 []。
- 用户大纲只有两级时,sub_points 留空 []。
【输出要求】
- 只输出一个 JSON 对象,不要解释文字,不要 markdown 代码块,必须用英文双引号。
- material_ids 一律输出 [](此步不关联素材)。
- 结构:{"title": "...", "sections": [{"section_title": "...", "key_points": [{"point_title": "...", "sub_points": ["..."]}], "material_ids": []}]}
- 用户大纲若包含一个总标题,放进 title;没有就留空字符串。"""
OUTLINE_NORMALIZE_USER = """用户粘贴的大纲原文:
{raw_outline}
请逐条保留原文,仅判断层级,输出结构化大纲 JSON。"""
# ── 用户自带大纲:素材关联(结构锁定,只挑 material_ids)──────────────────────────
OUTLINE_MATERIAL_ASSIGN_SYSTEM = """你是一名资料编辑。
给定一份【已确定的文章大纲】(章节顺序固定)和一批检索到的素材,你的任务是为每个章节
挑选最相关的素材编号。
【规则】
- 不要修改大纲的任何标题或结构,你只输出每个章节对应的素材编号列表。
- 严格按大纲给出的章节顺序输出,一个章节对应一个编号数组;该章节没有相关素材就给 []。
- material_id 必须来自给定素材清单里出现过的编号,不要编造。
- 一条素材可以分配给多个章节,也可以一个都不分配。
【输出要求】
- 只输出一个 JSON 对象,不要解释文字,不要 markdown 代码块,必须用英文双引号。
- 结构:{"assignments": [["m001","m002"], [], ["m003"]]},数组长度等于章节数、顺序一致。"""
OUTLINE_MATERIAL_ASSIGN_USER = """写作意图:{user_intent}
文章大纲(章节按顺序编号):
{sections_list}
可用素材:
{materials_summary}
请按章节顺序,为每个章节挑选相关的 material_id,输出 assignments 数组。"""
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
@@ -16,6 +16,7 @@
from __future__ import annotations
import logging
import time
from typing import Callable, List, Optional
from langchain_core.messages import HumanMessage, SystemMessage
@@ -26,6 +27,7 @@ from ..nodes._utils import (
_content_to_text,
get_model,
parse_json,
reasoning_delta,
safe_get_stream_writer,
)
@@ -127,12 +129,26 @@ class SkillPicker:
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
# list_skills 之后立刻发一个"分析中"提示步骤,避免 LLM 第一个 chunk 到来前
# 用户看到空窗期(某些模型 astream 在头部 buffer 时间较长)
self._emit_step({
"type": "match_skills",
"message": "正在让模型从候选技能中筛选最合适的…",
})
# 流式 LLM 调用:与 intent_parser / writer_outline 同套机制
writer = safe_get_stream_writer()
parser = ThinkTagParser()
full_text = ""
_chunk_count = {"n": 0}
def _emit_chunk(text: str, is_thinking: bool) -> None:
# 只在首个 chunk 打一条 timing,避免逐 token 刷屏;用来和 skill_search_step
# 的 [skill-timing] 对比"选择打字"与"步骤事件"谁先到。
if _chunk_count["n"] == 0:
logger.info("[skill-timing] t=%.3f first skill_select_chunk", time.monotonic())
_chunk_count["n"] += 1
writer({
"type": "skill_select_chunk",
"text": text,
@@ -140,6 +156,10 @@ class SkillPicker:
"agent_name": "素材收集专家",
})
# 立刻推一个空 chunk,让前端 streaming view 立即挂载(占位 + 闪烁光标),
# 即使后续 LLM 首个 chunk 到达较慢,UI 也不再静默
_emit_chunk("", False)
try:
model = get_model(self._config, model_name=self._model_name)
async for chunk in model.astream([
@@ -152,6 +172,10 @@ class SkillPicker:
max_select=self._max_select,
)),
]):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
_emit_chunk(reasoning, True)
delta = _content_to_text(chunk.content)
if not delta:
continue
@@ -125,6 +125,9 @@ class BlockedSection(TypedDict):
class AIWritingState(TypedDict):
# 写作配置
user_intent: str
# 用户在写作表单粘贴的「已有大纲」原文(可选)。非空时 writer_outline 跳过 LLM 规划,
# 直接解析这份大纲并按用户结构生成,做到「完全按我的大纲写」。留空走正常自动规划。
user_outline_raw: str
article_type: str
target_audience: str
word_count_target: int
@@ -132,6 +135,17 @@ class AIWritingState(TypedDict):
keyword_count: int
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
writing_mode: str
# 素材来源:knowledge_base=知识库(默认 web/intranet 检索)/ skill=技能检索。
# 用户在写作表单选择,首次检索时由 researcher 节点据此决定检索方式(缺省 knowledge_base)。
material_source: str
# 用户在写作表单选定的主模型。大纲/写作/审核严格使用它(不回退默认)。
# 由 intent_parser 在首次 invocation 从 configurable.model_name 捕获进 state,
# 经 checkpoint 持久化 —— interrupt/resume 是新 invocation、其 config 不带原
# model_name,存进 state 才能让 resume 后才运行的节点拿到用户模型。
model_name: str
# 笔记来源(可选):传入后 researcher 阶段会额外从笔记检索素材
notebook_search_space_id: Optional[int]
notebook_source_ids: Optional[List[int]]
# Agent A 输出
material_package: Optional[MaterialPackage]
@@ -11,6 +11,7 @@ from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
from deerflow.agents.middlewares.setup_writing_approval_middleware import SetupWritingApprovalMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
@@ -25,6 +26,9 @@ from deerflow.models import create_chat_model
logger = logging.getLogger(__name__)
# 写作配置 Q&A:意图澄清最多几轮(一条用户消息 = 一轮),到顶强制提交配置卡。
WRITING_SETUP_MAX_ROUNDS = 3
def _get_runtime_config(config: RunnableConfig) -> dict:
"""Merge legacy configurable options with LangGraph runtime context."""
@@ -272,6 +276,7 @@ def _build_middlewares(
# Add TodoList middleware if plan mode is enabled
is_plan_mode = cfg.get("is_plan_mode", False)
is_bootstrap = cfg.get("is_bootstrap", False)
is_writing_setup = cfg.get("is_writing_setup", False)
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
if todo_list_middleware is not None:
middlewares.append(todo_list_middleware)
@@ -280,18 +285,28 @@ def _build_middlewares(
if resolved_app_config.token_usage.enabled:
middlewares.append(TokenUsageMiddleware())
# Add TitleMiddleware
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
# Add TitleMiddleware — but skip it for scheduled runs. Each scheduled run
# executes on a throwaway thread whose title is never shown to a user, and
# generating one would fire an extra LLM call on the *default* model
# (config.title.model_name is usually unset → models[0]), which is not the
# model the task selected. Skipping it keeps scheduled/HTML runs on exactly
# the chosen model.
if not cfg.get("is_scheduled_run", False):
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
# Add MemoryMiddleware (after TitleMiddleware) — skip in bootstrap mode (agent creation flow)
memory_injection_enabled = cfg.get("memory_injection_enabled", True)
if not is_bootstrap:
if not is_bootstrap and not is_writing_setup:
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
# Add ViewImageMiddleware only if the current model supports vision.
# Use the resolved runtime model_name from make_lead_agent to avoid stale config values.
model_config = resolved_app_config.get_model_config(model_name) if model_name else None
if model_config is not None and model_config.supports_vision:
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
# ingests raw image data). In delegated mode the view_image tool already
# returns a text description and stores nothing in `viewed_images`, so the
# middleware would otherwise inject a spurious "No images have been viewed."
# message. Use the resolved runtime model_name to avoid stale config values.
from deerflow.models.vision import is_delegated_vision, vision_enabled
if vision_enabled(model_name, app_config=resolved_app_config) and not is_delegated_vision(app_config=resolved_app_config):
middlewares.append(ViewImageMiddleware())
# Add DeferredToolFilterMiddleware to hide deferred tool schemas from model binding
@@ -314,6 +329,7 @@ def _build_middlewares(
middlewares.extend(custom_middlewares)
middlewares.append(SetupAgentApprovalMiddleware())
middlewares.append(SetupWritingApprovalMiddleware())
# SkillStopMiddleware — intercepts skill invocations and halts execution
# when a configured skill is about to run. Request-level options
@@ -361,7 +377,7 @@ def make_lead_agent(config: RunnableConfig):
def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
# Lazy import to avoid circular dependency
from deerflow.tools import get_available_tools
from deerflow.tools.builtins import setup_agent
from deerflow.tools.builtins import ask_clarification_tool, setup_agent, setup_writing
cfg = _get_runtime_config(config)
resolved_app_config = app_config
@@ -372,6 +388,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
is_plan_mode = cfg.get("is_plan_mode", False)
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
is_bootstrap = cfg.get("is_bootstrap", False)
# 写作配置 Q&A:与 is_bootstrap 同构的另一种「最小化 bootstrap」——挂 setup_writing 工具、
# 走写作配置专用 prompt,只负责帮用户把写作表单聊清并提交,不真正开始写作。
is_writing_setup = cfg.get("is_writing_setup", False)
writing_setup_article_types = cfg.get("valid_article_types") if is_writing_setup else None
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
writing_mode = bool(cfg.get("writing_mode", False))
@@ -382,7 +402,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
requested_excluded_tools = cfg.get("excluded_tools") or None
try:
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
agent_config = load_agent_config(agent_id) if not is_bootstrap and not is_writing_setup else None
except FileNotFoundError:
logger.warning("Agent directory not found for '%s'; running with default config", agent_id)
agent_config = None
@@ -479,6 +499,25 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
}
)
if is_writing_setup:
# 写作配置 Q&A:最小化 agent —— 只挂 setup_writing(提交配置)+ ask_clarification(弹澄清卡)。
# 关键:绝不给全套工具、绝不复用 lead 超级体提示,否则模型会把「新能源汽车市场分析」之类
# 长得像任务的输入直接做掉,而不是转成写作配置提交。
# 末尾追加 WritingSetupRoundCapMiddleware:澄清最多 3 轮,到顶用 tool_choice 强制提交配置卡。
from deerflow.agents.lead_agent.prompt import build_writing_setup_prompt
from deerflow.agents.middlewares.writing_setup_round_cap_middleware import WritingSetupRoundCapMiddleware
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=resolved_app_config),
tools=[setup_writing, ask_clarification_tool],
middleware=[
*_build_middlewares(config, model_name=model_name, app_config=resolved_app_config),
WritingSetupRoundCapMiddleware(max_rounds=WRITING_SETUP_MAX_ROUNDS),
],
system_prompt=build_writing_setup_prompt(writing_setup_article_types),
state_schema=ThreadState,
)
if is_bootstrap:
# Special bootstrap agent with minimal prompt for initial custom agent creation flow
return create_agent(
@@ -703,14 +703,14 @@ def _get_cached_skills_prompt_section(
display_pattern = (
"""
**Skill Result Display Pattern:**
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
**Skill Result Display Pattern:**
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
"""
if display_signature
else ""
@@ -904,6 +904,46 @@ def _build_bootstrap_section() -> str:
"""
def build_writing_setup_prompt(article_types: list[dict] | None = None) -> str:
"""写作配置 Q&A 的**独立**系统提示(不复用 lead-agent 超级体提示)。
关键:写作配置助手绝不能像 lead agent 那样「自己把任务做掉」——用户输入常常长得像一个
可执行任务(如「新能源汽车市场分析」),若复用 lead 提示 + 全套工具,模型会直接去研究/写
分析。这里用一份职责单一、口吻强硬的独立提示,配合「只挂 setup_writing 一个工具」,把模型
死死框在「聊清需求 → 调 setup_writing」上。
"""
if article_types:
type_lines = "\n".join(
f" - `{t.get('key', '')}`:{t.get('label', '')}" for t in article_types if t.get("key")
)
type_block = f"【可选文章类型】article_type 只能填下列 key 之一:\n{type_lines}"
else:
type_block = "【文章类型】若不确定 article_type,可留空,交给用户在表单里选。"
return f"""你是「AI 写作」功能的**配置助手**。你的唯一职责:通过简短对话把用户的写作需求聊清楚,然后调用 `setup_writing` 工具把写作配置提交给用户确认。提交后由用户在表单里应用、再点「开始写作」。
【铁律——最高优先级】
- 你**绝对不撰写**文章正文、分析内容、大纲内容或任何成稿文字。即使用户的输入看起来就是一个写作任务(例如「新能源汽车市场分析」「帮我写一篇……」),你也**只把它当作"要配置的写作目标"**,绝不替用户动笔、绝不直接完成这篇写作。
- 你只有两个工具:① `ask_clarification`(向用户提问澄清,需要时用);② `setup_writing`(提交写作配置)。**没有任何其它工具**,不要做检索、不要做研究、不要假装调用别的工具。
- 不要在回复里输出文章段落、要点清单或大纲(用户主动粘贴的大纲除外——原样放进 `outline` 参数)。
【工作流程】(与用户交互确认,但最多 3 轮内必须提交)
1. 读用户输入。当关键项(文章类型、目标字数、目标读者)缺失或模糊时,调用 `ask_clarification` 向用户提问——**优先给出 `options` 选项**让用户点选(例如目标字数给「800 / 1500 / 3000」,读者给「普通读者 / 行业专家 / 政策决策者」)。一次可就 1-2 个独立的关键项各发一个 ask_clarification。
2. 与用户的澄清来回**最多 2 次**。一旦关键信息明确、或已经问过 2 次,就**立即进入第 3 步提交**,缺失字段用合理默认值,绝不无休止追问。
3. 调用 `setup_writing` 提交配置:
- `user_intent`:精炼成一句话的写作目标(必填)。
- `article_type`:见下方可选类型;不确定就留空。
- `word_count_target`:正整数;用户没明说就给合理默认(如 800)。
- `target_audience`:如「普通读者」「行业专家」;没提就填「普通读者」。
- `writing_mode`:用户强调「严格依据素材/不要编造」→ `strict`,否则 `loose`。
- `material_source`:**默认 `knowledge_base`(知识库)**;只有当用户明确要求用「技能/技能检索」时才填 `skill`,否则一律 `knowledge_base`。
- `outline`:用户提供的大纲原文(保留其层级与标题),否则留空。
4. 调用 `setup_writing` 前,用**一句话**说明你将提交的配置要点;调用之后不要再输出任何文章内容。
{type_block}
"""
def apply_prompt_template(
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
@@ -996,16 +1036,16 @@ Writing mode is enabled for this run.
network_policy_section = _build_network_policy_section()
rag_mode_section = (
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
When a search, knowledge, skill, or MCP tool returns a result list:
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
</rag_citation_mode>
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
When a search, knowledge, skill, or MCP tool returns a result list:
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
</rag_citation_mode>
"""
if rag_mode_enabled
else ""
)
@@ -67,6 +67,10 @@ class SetupAgentApprovalMiddleware(AgentMiddleware[SetupAgentApprovalMiddlewareS
args = setup_call.get("args") or {}
tool_call_id = setup_call.get("id") or ""
if isinstance(args.get("skills"), list) and len(args["skills"]) > 5:
args = {**args, "skills": args["skills"][:5]}
payload = self._build_payload(args)
logger.info("Setup agent approval requested before writing agent files")
@@ -0,0 +1,133 @@
"""Approval middleware for the AI-writing config setup tool.
与 ``SetupAgentApprovalMiddleware`` 同构:模型在「写作配置 Q&A」里调用 ``setup_writing``
时,于 ``after_model`` 拦截,附上一条结构化审批 ToolMessage(建议的写作配置打包在
``additional_kwargs["setup_writing_approval"]``)并 ``jump_to="end"`` 立即结束本轮运行——
前端据此渲染审批卡,用户点「应用到表单」后把这些字段回填到左侧写作表单。
"""
from __future__ import annotations
import logging
from hashlib import sha256
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware, hook_config
from langchain_core.messages import AIMessage, ToolMessage
from langgraph.runtime import Runtime
logger = logging.getLogger(__name__)
class SetupWritingApprovalMiddlewareState(AgentState):
"""Compatible with the DeerFlow thread state schema."""
pass
class SetupWritingApprovalMiddleware(AgentMiddleware[SetupWritingApprovalMiddlewareState]):
"""Stop before ``setup_writing`` and ask the user to apply the proposed config.
See ``SetupAgentApprovalMiddleware`` for the rationale behind intercepting in
``after_model`` + ``jump_to="end"`` (robust against parallel tool calls).
"""
state_schema = SetupWritingApprovalMiddlewareState
def _is_approved(self, runtime: Runtime) -> bool:
context = getattr(runtime, "context", None) or {}
if not isinstance(context, dict):
return False
return context.get("setup_writing_approved") is True
def _stable_message_id(self, tool_call_id: str, args: dict[str, Any]) -> str:
if tool_call_id:
return f"setup-writing-approval:{tool_call_id}"
digest = sha256(repr(sorted(args.items())).encode("utf-8")).hexdigest()[:16]
return f"setup-writing-approval:{digest}"
def _coerce_word_count(self, raw: Any) -> int | None:
if isinstance(raw, bool): # bool 是 int 子类,先排除
return None
if isinstance(raw, int):
return raw if raw > 0 else None
if isinstance(raw, float):
return int(raw) if raw > 0 else None
if isinstance(raw, str):
import re
m = re.search(r"\d[\d,]*", raw)
if m:
try:
value = int(m.group().replace(",", ""))
return value if value > 0 else None
except ValueError:
return None
return None
def _build_payload(self, args: dict[str, Any]) -> dict[str, Any]:
# 素材来源:仅当模型明确给出 "skill" 时才用技能检索;用户没明确要求就一律默认知识库。
material_source = (
"skill" if str(args.get("material_source") or "").strip() == "skill" else "knowledge_base"
)
return {
"user_intent": str(args.get("user_intent") or "").strip(),
"article_type": str(args.get("article_type") or "").strip(),
"word_count_target": self._coerce_word_count(args.get("word_count_target")),
"target_audience": str(args.get("target_audience") or "").strip(),
"writing_mode": str(args.get("writing_mode") or "").strip(),
"material_source": material_source,
"outline": str(args.get("outline") or "").strip(),
}
def _intercept(self, last: AIMessage) -> dict[str, Any] | None:
"""Return a state update if setup_writing must be gated; None to passthrough."""
tool_calls = list(last.tool_calls or [])
setup_call = next((tc for tc in tool_calls if tc.get("name") == "setup_writing"), None)
if setup_call is None:
return None
args = setup_call.get("args") or {}
tool_call_id = setup_call.get("id") or ""
payload = self._build_payload(args)
logger.info("Setup writing approval requested before applying writing config")
approval_msg = ToolMessage(
id=self._stable_message_id(tool_call_id, args),
content="请确认是否把这份写作配置应用到表单。",
tool_call_id=tool_call_id,
name="setup_writing",
additional_kwargs={"setup_writing_approval": payload},
)
return {
"messages": [approval_msg],
"jump_to": "end",
}
@hook_config(can_jump_to=["end"])
@override
def after_model(
self,
state: SetupWritingApprovalMiddlewareState,
runtime: Runtime,
) -> dict[str, Any] | None:
if self._is_approved(runtime):
return None
messages = state.get("messages") or []
if not messages:
return None
last = messages[-1]
if not isinstance(last, AIMessage):
return None
return self._intercept(last)
@hook_config(can_jump_to=["end"])
@override
async def aafter_model(
self,
state: SetupWritingApprovalMiddlewareState,
runtime: Runtime,
) -> dict[str, Any] | None:
return self.after_model(state, runtime)
@@ -23,6 +23,7 @@ class TitleMiddlewareState(AgentState):
"""Compatible with the `ThreadState` schema."""
title: NotRequired[str | None]
title_provisional: NotRequired[bool]
class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
@@ -61,16 +62,37 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
return ""
def _should_generate_title(self, state: TitleMiddlewareState) -> bool:
"""Check if we should generate a title for this thread."""
def _should_set_provisional_title(self, state: TitleMiddlewareState) -> bool:
"""Check if we should set an instant, question-derived placeholder title.
Runs in ``before_model`` so the thread shows a sensible title the moment
the user sends their first message — eliminating the "untitled" window
while the (possibly long) first answer streams. No LLM call, no latency.
"""
config = self._get_title_config()
if not config.enabled:
return False
# Check if thread already has a title in state
# Already has a title (provisional or final) — nothing to do.
if state.get("title"):
return False
# Only on the very first turn, as soon as the user message arrives.
messages = state.get("messages", [])
user_messages = [m for m in messages if m.type == "human"]
return len(user_messages) == 1
def _should_generate_title(self, state: TitleMiddlewareState) -> bool:
"""Check if we should generate the final LLM title for this thread."""
config = self._get_title_config()
if not config.enabled:
return False
# Skip only if we already have a *finalized* title. A provisional
# placeholder set in before_model should still be upgraded by the LLM.
if state.get("title") and not state.get("title_provisional"):
return False
# Check if this is the first turn (has at least one user message and one assistant response)
messages = state.get("messages", [])
if len(messages) < 2:
@@ -146,13 +168,27 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
config["tags"] = [*(config.get("tags") or []), "middleware:title"]
return config
def _provisional_title_result(self, state: TitleMiddlewareState) -> dict | None:
"""Set an instant placeholder title from the first user message (no LLM)."""
if not self._should_set_provisional_title(state):
return None
messages = state.get("messages", [])
user_msg_content = next((m.content for m in messages if m.type == "human"), "")
user_msg = self._normalize_content(user_msg_content)
if not user_msg.strip():
return None
title = self._fallback_title(user_msg)
logger.debug("TitleMiddleware: set provisional title %r before first model call", title)
return {"title": title, "title_provisional": True}
def _generate_title_result(self, state: TitleMiddlewareState) -> dict | None:
"""Generate a local fallback title without blocking on an LLM call."""
if not self._should_generate_title(state):
return None
_, user_msg = self._build_title_prompt(state)
return {"title": self._fallback_title(user_msg)}
return {"title": self._fallback_title(user_msg), "title_provisional": False}
def _force_disable_thinking(self, model: object) -> None:
"""Force-disable thinking on the title model via `extra_body` overrides.
@@ -204,10 +240,18 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
response = await model.ainvoke(prompt, config=self._get_runnable_config())
title = self._parse_title(response.content)
if title:
return {"title": title}
return {"title": title, "title_provisional": False}
except Exception:
logger.debug("Failed to generate async title; falling back to local title", exc_info=True)
return {"title": self._fallback_title(user_msg)}
return {"title": self._fallback_title(user_msg), "title_provisional": False}
@override
def before_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
return self._provisional_title_result(state)
@override
async def abefore_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
return self._provisional_title_result(state)
@override
def after_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
@@ -18,6 +18,12 @@ logger = logging.getLogger(__name__)
_OUTLINE_PREVIEW_LINES = 5
# Image files are understood via the `view_image` tool (which routes to the
# configured vision model), never by reading their bytes. Listing them with the
# generic read_file/grep guidance made the agent try to parse image binaries
# with bash/python, which fails and wastes turns.
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
def _extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]:
"""Return the document outline and fallback preview for *file_path*.
@@ -88,6 +94,16 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
size_str = f"{size_kb:.1f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB"
lines.append(f"- {file['filename']} ({size_str})")
lines.append(f" Path: {file['path']}")
# Images must be understood through the vision model, not by reading bytes.
extension = str(file.get("extension") or Path(file["filename"]).suffix).lower()
if extension in _IMAGE_EXTENSIONS:
lines.append(f" This is an image. To understand it, call the `view_image` tool with image_path='{file['path']}' (optionally pass a `question`).")
lines.append(" The configured vision model will analyze it and return a description — trust that result.")
lines.append(" Do NOT read, open, or parse the image with read_file, grep, bash, python, PIL, or any byte-level inspection; those cannot interpret image content.")
lines.append("")
return
outline = file.get("outline") or []
if outline:
truncated = outline[-1].get("truncated", False)
@@ -139,6 +139,12 @@ class ViewImageMiddleware(AgentMiddleware[ViewImageMiddlewareState]):
if not messages:
return False
# Nothing to inject when no image data was captured (e.g. delegated
# recognition handled the image in the tool and stored no base64).
# Avoids injecting a useless "No images have been viewed." message.
if not state.get("viewed_images"):
return False
# Get the last assistant message
last_assistant_msg = self._get_last_assistant_message(messages)
if not last_assistant_msg:
@@ -0,0 +1,72 @@
"""写作配置 Q&A 的「轮数上限」中间件。
意图阶段允许与用户来回澄清确认,但**最多 N 轮后必须出结果**:到上限时,在模型请求里
注入强制指令 + 用 ``tool_choice`` 锁定 ``setup_writing``,逼模型立刻提交配置卡,不再提问。
「一轮」= 一条用户消息(HumanMessage)。达到 ``max_rounds`` 条用户消息时强制提交。
"""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import HumanMessage, SystemMessage
class WritingSetupRoundCapMiddleware(AgentMiddleware[AgentState]):
"""达到澄清轮数上限时,强制模型调用 ``setup_writing`` 提交。
软约束(注入 SystemMessage)+ 硬约束(``tool_choice`` 锁定工具)双保险:即使个别模型
无视提示,``tool_choice`` 也会逼它产出一次 ``setup_writing`` 调用。
"""
def __init__(self, max_rounds: int = 3, tool_name: str = "setup_writing") -> None:
super().__init__()
self._max_rounds = max(1, max_rounds)
self._tool_name = tool_name
def _should_force(self, messages: list) -> bool:
human_count = sum(1 for m in messages if isinstance(m, HumanMessage))
return human_count >= self._max_rounds
def _reminder(self) -> SystemMessage:
return SystemMessage(
content=(
f"【系统强制】意图澄清已达 {self._max_rounds} 轮上限。"
"现在必须立即调用 setup_writing 提交写作配置,禁止再向用户提任何问题。"
"缺失或不确定的字段一律用合理默认值:"
"word_count_target 默认 800、target_audience 默认「普通读者」、"
"writing_mode 默认 loose、material_source 默认 knowledge_base、"
"article_type 不确定就留空。"
)
)
def _patch(self, request: ModelRequest) -> ModelRequest:
return request.override(
messages=[*request.messages, self._reminder()],
tool_choice=self._tool_name,
)
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
if self._should_force(request.messages):
request = self._patch(request)
return handler(request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
if self._should_force(request.messages):
request = self._patch(request)
return await handler(request)
@@ -69,6 +69,10 @@ class ThreadState(AgentState):
sandbox: NotRequired[SandboxState | None]
thread_data: NotRequired[ThreadDataState | None]
title: NotRequired[str | None]
# True while `title` is a provisional, question-derived placeholder that the
# async LLM pass has not yet finalized. Lets TitleMiddleware show a title
# instantly (no "untitled" window) and upgrade it once the answer is ready.
title_provisional: NotRequired[bool]
artifacts: Annotated[list[str], merge_artifacts]
todos: NotRequired[list | None]
uploaded_files: NotRequired[list[dict] | None]
@@ -15,6 +15,7 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
from deerflow.config.run_events_config import RunEventsConfig
@@ -32,6 +33,7 @@ from deerflow.config.title_config import TitleConfig, load_title_config_from_dic
from deerflow.config.token_usage_config import TokenUsageConfig
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
from deerflow.config.vision_config import VisionConfig
load_dotenv()
@@ -104,6 +106,8 @@ class AppConfig(BaseModel):
log_level: str = Field(default="info", description="Logging level for deerflow and app modules (debug/info/warning/error); third-party libraries are not affected")
token_usage: TokenUsageConfig = Field(default_factory=TokenUsageConfig, description="Token usage tracking configuration")
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
sandbox: SandboxConfig = Field(description="Sandbox configuration")
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
@@ -62,6 +62,52 @@ class SkillDisplayTableColumn(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayResultSetConfig(BaseModel):
"""Display adapter for one result array inside a skill response."""
key: str = Field(description="Stable key for this result set")
label: str | None = Field(default=None, description="Human-readable result set label")
enabled: bool = Field(default=True, description="Whether this result set is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="citation",
description="How the frontend should render this result set",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="singleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayConfig(BaseModel):
"""Frontend display adapter for structured skill results."""
@@ -103,6 +149,11 @@ class SkillDisplayConfig(BaseModel):
description="Columns used when mode is table",
alias="tableColumns",
)
result_sets: list[SkillDisplayResultSetConfig] = Field(
default_factory=list,
description="Optional per-result-array display adapters",
alias="resultSets",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
@@ -0,0 +1,83 @@
from pydantic import BaseModel, Field
# Fallback shared password for the username-login gate. Used when
# ``UsernameLoginConfig.password`` is left empty but ``require_password`` is on.
DEFAULT_USERNAME_LOGIN_PASSWORD = "123ewq"
class UsernameLoginConfig(BaseModel):
"""Gate for the passwordless ``POST /api/v1/auth/login/username`` flow.
By default this flow logs a user in (auto-registering if needed) from a
username alone — convenient, but it lets anyone sign in as any username.
Turning ``require_password`` on forces callers to additionally pass a
shared secret as ``?password=...``; mismatches are rejected with 401.
"""
require_password: bool = Field(
default=False,
description="When true, login/username requires a matching ?password= query value.",
)
password: str = Field(
default="",
description=f"Shared secret for the username-login gate. Empty falls back to '{DEFAULT_USERNAME_LOGIN_PASSWORD}'.",
)
@property
def effective_password(self) -> str:
"""The password actually checked at login time (config value or default)."""
return self.password or DEFAULT_USERNAME_LOGIN_PASSWORD
class TokenLoginConfig(BaseModel):
"""Token-exchange login.
The frontend passes an upstream access token via the URL. The gateway calls
``token_info_url`` with that token placed directly in the
``Authorization: Bearer <token>`` header, reads ``data.username`` from the
response, then signs the matching local user in (auto-registering if the
username is not yet in the database).
"""
enabled: bool = Field(
default=False,
description="Master switch for the POST /api/v1/auth/login/token endpoint.",
)
token_info_url: str = Field(
default="",
description="Upstream endpoint that resolves an access token to a username.",
)
service_authorization: str = Field(
default="Bearer admin",
description=(
"Deprecated/unused: the Authorization header is now built from the "
"caller's token as 'Bearer <token>'. Retained for backward compat."
),
)
timeout_seconds: float = Field(
default=10.0,
gt=0,
description="HTTP timeout (seconds) for the token_info_url call.",
)
verify_ssl: bool = Field(
default=True,
description=(
"Whether to verify the upstream TLS certificate. Set false for "
"internal/UAT hosts with self-signed or private-CA certs. Leave "
"true in production."
),
)
ca_cert_path: str = Field(
default="",
description=(
"Optional path to a CA bundle (PEM) used to verify the upstream "
"certificate. Takes precedence over verify_ssl when set."
),
)
class AuthLoginConfig(BaseModel):
"""Container for the custom login modes layered on top of the base auth."""
username_login: UsernameLoginConfig = Field(default_factory=UsernameLoginConfig)
token_login: TokenLoginConfig = Field(default_factory=TokenLoginConfig)
@@ -0,0 +1,29 @@
from pydantic import BaseModel, Field
# Default instruction sent to the vision model when the agent does not provide a
# more specific question. Kept in Chinese to match the product's primary locale.
DEFAULT_VISION_PROMPT = "请仔细观察这张图片,尽可能详细、准确地描述其中的全部内容,包括文字、图表、物体、场景与布局等关键信息。"
class VisionConfig(BaseModel):
"""Configuration for image recognition (识图).
When ``model_name`` names a model from ``models[]``, the lead agent's
``view_image`` tool delegates image understanding to that dedicated model
and returns a text description — so a text-only chat model can still "see"
uploaded images. When ``model_name`` is unset, vision falls back to the main
chat model, which only works when that model has ``supports_vision: true``.
"""
model_name: str | None = Field(
default=None,
description="Name of the model (from models[]) used for image recognition. When unset, falls back to the main chat model if it supports vision.",
)
prompt: str = Field(
default=DEFAULT_VISION_PROMPT,
description="Default instruction for the vision model when no per-call question is given.",
)
max_tokens: int | None = Field(
default=None,
description="Optional override for the vision model's max output tokens; falls back to the model's own config when unset.",
)
@@ -1,3 +1,10 @@
from .factory import create_chat_model
from .vision import is_delegated_vision, recognize_image, resolve_vision_model_name, vision_enabled
__all__ = ["create_chat_model"]
__all__ = [
"create_chat_model",
"is_delegated_vision",
"recognize_image",
"resolve_vision_model_name",
"vision_enabled",
]
@@ -0,0 +1,121 @@
"""Image recognition (识图) via a configurable vision model.
When ``config.yaml`` defines ``vision.model_name``, the lead agent's
``view_image`` tool delegates image understanding to that dedicated model and
returns a text description — so a text-only chat model can still "see" uploaded
images. When no dedicated model is configured, vision falls back to the main
chat model, which only works when that model has ``supports_vision: true``.
"""
import logging
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.vision_config import DEFAULT_VISION_PROMPT
logger = logging.getLogger(__name__)
def _configured_vision_model(config: AppConfig) -> str | None:
"""Return a valid dedicated vision model name from config, or None."""
vision_cfg = getattr(config, "vision", None)
name = getattr(vision_cfg, "model_name", None) if vision_cfg else None
if not name:
return None
if config.get_model_config(name) is None:
logger.warning("vision.model_name %r is not defined in models[]; ignoring", name)
return None
return name
def resolve_vision_model_name(model_name: str | None, *, app_config: AppConfig | None = None) -> str | None:
"""Resolve which model should handle image recognition for this run.
Priority:
1. A dedicated ``vision.model_name`` configured in ``config.yaml``.
2. The main chat model, only if it declares ``supports_vision: true``.
3. ``None`` — vision is unavailable.
"""
config = app_config or get_app_config()
configured = _configured_vision_model(config)
if configured:
return configured
if model_name:
mc = config.get_model_config(model_name)
if mc is not None and mc.supports_vision:
return model_name
return None
def vision_enabled(model_name: str | None, *, app_config: AppConfig | None = None) -> bool:
"""Whether the ``view_image`` tool/middleware should be available this run."""
return resolve_vision_model_name(model_name, app_config=app_config) is not None
def is_delegated_vision(*, app_config: AppConfig | None = None) -> bool:
"""Whether image recognition is delegated to a dedicated vision model.
True when ``vision.model_name`` is configured and valid — in that case
``view_image`` calls the dedicated model and returns a text description
instead of injecting raw image data into the main chat model.
"""
config = app_config or get_app_config()
return _configured_vision_model(config) is not None
def _extract_text(message) -> str:
"""Flatten an LLM response message into plain text."""
content = getattr(message, "content", message)
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict) and block.get("type") == "text":
parts.append(str(block.get("text", "")))
return "\n".join(p for p in parts if p).strip()
return str(content).strip()
def recognize_image(
image_base64: str,
mime_type: str,
*,
question: str | None = None,
app_config: AppConfig | None = None,
) -> str:
"""Describe/analyze an image using the configured dedicated vision model.
Returns the model's text answer. Raises ``ValueError`` if no dedicated
vision model is configured, and propagates model/network errors to the
caller for handling.
"""
from langchain_core.messages import HumanMessage
from deerflow.models.factory import create_chat_model
config = app_config or get_app_config()
model_name = _configured_vision_model(config)
if model_name is None:
raise ValueError("No dedicated vision model is configured (vision.model_name).")
vision_cfg = getattr(config, "vision", None)
default_prompt = getattr(vision_cfg, "prompt", None) or DEFAULT_VISION_PROMPT
prompt = (question or "").strip() or default_prompt
extra: dict = {}
max_tokens = getattr(vision_cfg, "max_tokens", None) if vision_cfg else None
if max_tokens:
extra["max_tokens"] = max_tokens
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config, **extra)
message = HumanMessage(
content=[
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_base64}"}},
]
)
result = model.invoke([message])
return _extract_text(result)
@@ -0,0 +1,18 @@
"""Persistence for iframe embed session → thread mappings."""
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
from deerflow.persistence.embed_sessions.memory import MemoryEmbedSessionStore
from deerflow.persistence.embed_sessions.sql import EmbedSessionRepository
__all__ = [
"EmbedSessionStore",
"MemoryEmbedSessionStore",
"EmbedSessionRepository",
"make_embed_session_store",
]
def make_embed_session_store(session_factory):
if session_factory is None:
return MemoryEmbedSessionStore()
return EmbedSessionRepository(session_factory)
@@ -0,0 +1,18 @@
"""Abstract persistence API for iframe embed session → thread mappings."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class EmbedSessionStore(ABC):
@abstractmethod
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
"""Create or update the mapping for an embed session."""
raise NotImplementedError
@abstractmethod
async def get(self, session_id: str) -> dict[str, Any] | None:
"""Return ``{session_id, thread_id, created_at, updated_at}`` or None."""
raise NotImplementedError
@@ -0,0 +1,37 @@
"""In-memory ``EmbedSessionStore`` for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
class MemoryEmbedSessionStore(EmbedSessionStore):
def __init__(self) -> None:
self._sessions: dict[str, dict[str, Any]] = {}
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
existing = self._sessions.get(session_id)
if existing is None:
record = {
"session_id": session_id,
"thread_id": thread_id,
"created_at": now,
"updated_at": now,
}
else:
record = {
**existing,
"thread_id": thread_id,
"updated_at": now,
}
self._sessions[session_id] = record
return copy.deepcopy(record)
async def get(self, session_id: str) -> dict[str, Any] | None:
record = self._sessions.get(session_id)
return copy.deepcopy(record) if record is not None else None
@@ -0,0 +1,21 @@
"""ORM model for iframe embed session → thread lookups."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class EmbedSessionRow(Base):
__tablename__ = "embed_sessions"
# Opaque id generated by the parent page and passed into the iframe URL.
session_id: Mapped[str] = mapped_column(String(128), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,49 @@
"""SQLAlchemy-backed embed session repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
def _row_to_dict(row: EmbedSessionRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class EmbedSessionRepository(EmbedSessionStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(EmbedSessionRow, session_id)
now = datetime.now(UTC)
if row is None:
row = EmbedSessionRow(
session_id=session_id,
thread_id=thread_id,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.thread_id = thread_id
row.updated_at = now
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def get(self, session_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(EmbedSessionRow, session_id)
return _row_to_dict(row) if row is not None else None
@@ -0,0 +1,23 @@
"""Generated HTML page + favorite persistence."""
from __future__ import annotations
from deerflow.persistence.html_pages.base import HtmlPageStore
from deerflow.persistence.html_pages.memory import MemoryHtmlPageStore
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.html_pages.sql import HtmlPageRepository
__all__ = [
"HtmlPageStore",
"HtmlPageRepository",
"MemoryHtmlPageStore",
"HtmlPageFavoriteRow",
"ScheduledHtmlPageRow",
"make_html_page_store",
]
def make_html_page_store(session_factory):
if session_factory is None:
return MemoryHtmlPageStore()
return HtmlPageRepository(session_factory)
@@ -0,0 +1,53 @@
"""Abstract persistence API for generated HTML pages and favorites."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class HtmlPageStore(ABC):
# --- Generated pages ---
@abstractmethod
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def get_page(self, page_id: str) -> dict[str, Any] | None:
"""Return a page by id without restricting by owner."""
raise NotImplementedError
@abstractmethod
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
"""Return the page produced by a given run, if any."""
raise NotImplementedError
@abstractmethod
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
"""Return the most recently created page for a task, if any."""
raise NotImplementedError
# --- Favorites ---
@abstractmethod
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
"""Return a favorite by id without restricting by owner."""
raise NotImplementedError
@abstractmethod
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
"""Return the user's existing favorite for a page, if any (dedup check)."""
raise NotImplementedError
@abstractmethod
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
raise NotImplementedError
@@ -0,0 +1,102 @@
"""In-memory HtmlPageStore for no-database / test deployments."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.html_pages.base import HtmlPageStore
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
class MemoryHtmlPageStore(HtmlPageStore):
def __init__(self) -> None:
self._pages: dict[str, dict[str, Any]] = {}
self._favorites: dict[str, dict[str, Any]] = {}
# --- Generated pages ---
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": data["id"],
"task_id": data["task_id"],
"run_id": data["run_id"],
"user_id": data["user_id"],
"title": data.get("title") or "",
"html_content": data.get("html_content") or "",
"html_file_path": data.get("html_file_path"),
"data_snapshot": data.get("data_snapshot") or {},
"prompt_snapshot": data.get("prompt_snapshot") or {},
"reference_favorite_id": data.get("reference_favorite_id"),
"style_summary": data.get("style_summary"),
"layout_summary": data.get("layout_summary"),
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
self._pages[row["id"]] = row
return copy.deepcopy(row)
async def get_page(self, page_id: str) -> dict[str, Any] | None:
row = self._pages.get(page_id)
return copy.deepcopy(row) if row is not None else None
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
matches = [r for r in self._pages.values() if r["run_id"] == run_id]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
matches = [r for r in self._pages.values() if r["task_id"] == task_id]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
# --- Favorites ---
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": data["id"],
"user_id": data["user_id"],
"page_id": data["page_id"],
"title": data.get("title") or "",
"description": data.get("description"),
"tags": data.get("tags") or [],
"html_snapshot": data.get("html_snapshot") or "",
"style_summary": data.get("style_summary"),
"layout_summary": data.get("layout_summary"),
"content_structure_summary": data.get("content_structure_summary"),
"reuse_prompt": data.get("reuse_prompt"),
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
self._favorites[row["id"]] = row
return copy.deepcopy(row)
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
matches = [r for r in self._favorites.values() if r["user_id"] == user_id]
matches.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in matches[: max(1, limit)]]
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
row = self._favorites.get(favorite_id)
return copy.deepcopy(row) if row is not None else None
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
for row in self._favorites.values():
if row["user_id"] == user_id and row["page_id"] == page_id:
return copy.deepcopy(row)
return None
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
row = self._favorites.get(favorite_id)
if row is None or row["user_id"] != user_id:
return False
del self._favorites[favorite_id]
return True
@@ -0,0 +1,65 @@
"""ORM models for generated HTML pages and user favorites.
These back the "HTML 页面定时任务" feature: a scheduled task of kind
``html_page`` renders a full HTML document each run. Every rendered page is
recorded in :class:`ScheduledHtmlPageRow`; a user may bookmark one into
:class:`HtmlPageFavoriteRow`, which also stores a reusable style/layout
summary so a later task can be generated "in the style of" the favorite.
Both tables are brand new, so ``Base.metadata.create_all`` creates them on the
next startup without touching any existing schema.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class ScheduledHtmlPageRow(Base):
"""One HTML page produced by a single run of an ``html_page`` task."""
__tablename__ = "scheduled_html_pages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), index=True)
run_id: Mapped[str] = mapped_column(String(64), index=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(255))
# Full HTML document — can be large, so LONGTEXT on MySQL.
html_content: Mapped[str] = mapped_column(PortableLongText())
html_file_path: Mapped[str | None] = mapped_column(Text)
data_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
prompt_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
reference_favorite_id: Mapped[str | None] = mapped_column(String(64), index=True)
style_summary: Mapped[str | None] = mapped_column(Text)
layout_summary: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class HtmlPageFavoriteRow(Base):
"""A user-bookmarked HTML page, with a reusable style/layout summary."""
__tablename__ = "html_page_favorites"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
page_id: Mapped[str] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(255))
description: Mapped[str | None] = mapped_column(Text)
tags_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list)
# Snapshot of the page HTML at favorite time, decoupled from the source run
# so deleting the run never breaks the favorite reference.
html_snapshot: Mapped[str] = mapped_column(PortableLongText())
style_summary: Mapped[str | None] = mapped_column(Text)
layout_summary: Mapped[str | None] = mapped_column(Text)
content_structure_summary: Mapped[str | None] = mapped_column(Text)
reuse_prompt: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,141 @@
"""SQLAlchemy repository for generated HTML pages and favorites."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.html_pages.base import HtmlPageStore
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
def _to_dict(row) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
# Normalize the JSON column names to API-friendly keys.
if "data_snapshot_json" in data:
data["data_snapshot"] = data.pop("data_snapshot_json") or {}
if "prompt_snapshot_json" in data:
data["prompt_snapshot"] = data.pop("prompt_snapshot_json") or {}
if "tags_json" in data:
data["tags"] = data.pop("tags_json") or []
return data
class HtmlPageRepository(HtmlPageStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# --- Generated pages ---
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
row = ScheduledHtmlPageRow(
id=data["id"],
task_id=data["task_id"],
run_id=data["run_id"],
user_id=data["user_id"],
title=data.get("title") or "",
html_content=data.get("html_content") or "",
html_file_path=data.get("html_file_path"),
data_snapshot_json=data.get("data_snapshot") or {},
prompt_snapshot_json=data.get("prompt_snapshot") or {},
reference_favorite_id=data.get("reference_favorite_id"),
style_summary=data.get("style_summary"),
layout_summary=data.get("layout_summary"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def get_page(self, page_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledHtmlPageRow, page_id)
return _to_dict(row) if row is not None else None
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(ScheduledHtmlPageRow)
.where(ScheduledHtmlPageRow.run_id == run_id)
.order_by(ScheduledHtmlPageRow.created_at.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(ScheduledHtmlPageRow)
.where(ScheduledHtmlPageRow.task_id == task_id)
.order_by(ScheduledHtmlPageRow.created_at.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
# --- Favorites ---
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
row = HtmlPageFavoriteRow(
id=data["id"],
user_id=data["user_id"],
page_id=data["page_id"],
title=data.get("title") or "",
description=data.get("description"),
tags_json=data.get("tags") or [],
html_snapshot=data.get("html_snapshot") or "",
style_summary=data.get("style_summary"),
layout_summary=data.get("layout_summary"),
content_structure_summary=data.get("content_structure_summary"),
reuse_prompt=data.get("reuse_prompt"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(HtmlPageFavoriteRow)
.where(HtmlPageFavoriteRow.user_id == user_id)
.order_by(HtmlPageFavoriteRow.created_at.desc())
.limit(max(1, limit))
)
return [_to_dict(row) for row in result.scalars()]
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(HtmlPageFavoriteRow, favorite_id)
return _to_dict(row) if row is not None else None
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(HtmlPageFavoriteRow)
.where(HtmlPageFavoriteRow.user_id == user_id, HtmlPageFavoriteRow.page_id == page_id)
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(HtmlPageFavoriteRow).where(
HtmlPageFavoriteRow.id == favorite_id,
HtmlPageFavoriteRow.user_id == user_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
@@ -0,0 +1,39 @@
"""Create ``embed_sessions`` table for iframe embed status polling.
Revision ID: 20260602_01
Revises: 20260528_02
Create Date: 2026-06-02
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260602_01"
down_revision = "20260528_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("embed_sessions"):
op.create_table(
"embed_sessions",
sa.Column("session_id", sa.String(128), primary_key=True),
sa.Column("thread_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_embed_sessions_thread_id", "embed_sessions", ["thread_id"])
def downgrade() -> None:
if _has_table("embed_sessions"):
op.drop_table("embed_sessions")
@@ -0,0 +1,49 @@
"""Add step3 column to roundtable_drafts.
Step 3「结果绘制」persistence: the generated report (HTML + metadata) is stored
inline in the draft so a reloaded draft can show the report without depending on
the ephemeral report thread. The column is LONGTEXT on MySQL (HTML can exceed the
64 KB TEXT cap) and plain TEXT (unbounded) elsewhere — matching ``PortableLongText``
on ``RoundtableDraftRow.step3``. ``create_all`` adds the column for brand-new DBs;
this revision patches DBs that already have the table.
Revision ID: 20260603_01
Revises: 20260602_01
Create Date: 2026-06-03
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260603_01"
down_revision = "20260602_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_drafts") and not _has_column("roundtable_drafts", "step3"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_drafts", sa.Column("step3", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_drafts", "step3"):
op.drop_column("roundtable_drafts", "step3")
@@ -22,6 +22,11 @@ from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.roundtable_drafts.model import (
RoundtableDraftRow,
RoundtableRecommendHistoryRow,
)
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import (
ScheduledTaskDeliveryProfileRow,
@@ -33,6 +38,7 @@ from deerflow.persistence.scheduled_tasks.model import (
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
@@ -45,6 +51,9 @@ __all__ = [
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",
"RoundtableChainRow",
"RoundtableDraftRow",
"RoundtableRecommendHistoryRow",
"RunEventRow",
"RunRow",
"ScheduledTaskDeliveryProfileRow",
@@ -56,6 +65,7 @@ __all__ = [
"SkillDisplayAdapterRow",
"TagAssignmentRow",
"TagRow",
"EmbedSessionRow",
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
@@ -0,0 +1,18 @@
"""Roundtable-planning business-chain persistence."""
from __future__ import annotations
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.roundtable_chains.sql import RoundtableChainRepository
__all__ = [
"RoundtableChainRepository",
"RoundtableChainRow",
"make_roundtable_chain_store",
]
def make_roundtable_chain_store(session_factory):
if session_factory is None:
return None
return RoundtableChainRepository(session_factory)
@@ -0,0 +1,44 @@
"""ORM model for roundtable-planning business chains.
A "business chain" (业务链条) is a named, reusable, *ordered* list of agents the
user assembles in the chain editor. In Step 2 the coordinator dispatches to the
seats following this order (see ``multi-agent-business-chain-dev.md``).
Persisted per user (like ``roundtable_drafts``) so it follows the account across
devices/sessions. ``seats`` is stored as a JSON array serialized into
``PortableLongText`` (MySQL ``LONGTEXT``) rather than ``PortableJSON`` for
headroom, mirroring the sibling roundtable persistence modules.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class RoundtableChainRow(Base):
__tablename__ = "roundtable_chains"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Human-readable chain title shown on the picker card.
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# Optional free-text description.
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# JSON: ordered [{"agent_id": str, "name": str, "description": str}, ...].
# seats[0] is the first seat to speak. Stored as LONGTEXT-backed JSON string.
seats: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,123 @@
"""SQLAlchemy-backed roundtable business-chain store.
The ``seats`` JSON column is stored as a serialized string in
``PortableLongText`` and round-tripped here with ``json.dumps`` / ``json.loads``
— same approach as ``roundtable_drafts`` / ``ai_writing_sessions``.
All write/read paths are scoped by ``user_id`` for ownership isolation; a row
that belongs to another user is treated as not found.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
_UNSET = object()
def _loads(value: Any) -> Any:
if not isinstance(value, str):
return value
if not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class RoundtableChainRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _to_dict(row: RoundtableChainRow) -> dict[str, Any]:
return {
"id": row.id,
"title": row.title,
"description": row.description,
"seats": _loads(row.seats) or [],
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
async def list_chains(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
stmt = (
select(RoundtableChainRow)
.where(RoundtableChainRow.user_id == user_id)
.order_by(RoundtableChainRow.updated_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row) for row in result.scalars()]
async def get_chain(self, chain_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableChainRow, chain_id)
if row is None or row.user_id != user_id:
return None
return self._to_dict(row)
async def create_chain(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = RoundtableChainRow(
id=data["id"],
user_id=user_id,
title=(data.get("title") or "").strip(),
description=(data.get("description") or None),
seats=_dumps(data.get("seats") or []),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def update_chain(
self,
chain_id: str,
user_id: str,
*,
title=_UNSET,
description=_UNSET,
seats=_UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableChainRow, chain_id)
if row is None or row.user_id != user_id:
return None
if title is not _UNSET and title is not None:
row.title = str(title).strip()
if description is not _UNSET:
row.description = description or None
if seats is not _UNSET:
row.seats = _dumps(seats)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def delete_chain(self, chain_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(RoundtableChainRow, chain_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
@@ -0,0 +1,22 @@
"""Roundtable-planning draft + recommendation-history persistence."""
from __future__ import annotations
from deerflow.persistence.roundtable_drafts.model import (
RoundtableDraftRow,
RoundtableRecommendHistoryRow,
)
from deerflow.persistence.roundtable_drafts.sql import RoundtableDraftRepository
__all__ = [
"RoundtableDraftRepository",
"RoundtableDraftRow",
"RoundtableRecommendHistoryRow",
"make_roundtable_draft_store",
]
def make_roundtable_draft_store(session_factory):
if session_factory is None:
return None
return RoundtableDraftRepository(session_factory)
@@ -0,0 +1,73 @@
"""ORM models for roundtable-planning drafts and recommendation history.
A "draft" is one full roundtable-planning session (Step 1 intent + Step 2
multi-agent roundtable), persisted per user so it follows the account across
devices/sessions — replacing the old browser-only ``localStorage`` storage.
``step1`` / ``step2`` are stored as JSON serialized into ``PortableLongText``
(MySQL ``LONGTEXT``) rather than ``PortableJSON`` (MySQL ``TEXT``, 64 KB cap),
because the Step 2 snapshot can carry a long roundtable transcript plus per-seat
stub messages that easily exceed 64 KB — mirroring ``ai_writing_sessions``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class RoundtableDraftRow(Base):
__tablename__ = "roundtable_drafts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Auto-derived task summary (objective → first user message → timestamp).
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# Furthest step reached when this snapshot was taken (1 | 2 | 3).
furthest_step: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
# JSON blobs matching the frontend Step1Snapshot / Step2Snapshot shapes.
step1: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
step2: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
# Step 3「结果绘制」snapshot: { html, generatedAt, model, summary? }. The full
# report HTML is stored inline so a reloaded draft can show the report without
# depending on the ephemeral report thread (LONGTEXT — HTML can exceed 64 KB).
step3: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class RoundtableRecommendHistoryRow(Base):
"""One agent-recommendation run, bound to a draft/session.
Lets the recommend dialog show "last recommendation for this session" on
open, while still letting the user re-analyze (which appends a new row).
"""
__tablename__ = "roundtable_recommend_history"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
draft_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Denormalized intent objective at recommend time (list display / context).
objective: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# How the run ended on the backend: "done" | "fallback" | "asking" | "error".
status: Mapped[str] = mapped_column(String(32), nullable=False, default="done")
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Natural-language rationale (may be long → LONGTEXT).
rationale: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# JSON: [{"agent_id": str, "reason": str}, ...]
picks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# JSON snapshot of the candidate pool at recommend time:
# [{"agent_id": str, "name": str, "description": str}, ...]
candidates: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,196 @@
"""SQLAlchemy-backed roundtable-planning draft + recommendation-history store.
JSON columns (``step1`` / ``step2`` / ``picks`` / ``candidates``) are stored as
serialized strings in ``PortableLongText`` and round-tripped here with
``json.dumps`` / ``json.loads`` — same approach as ``ai_writing_sessions`` — so
large Step 2 transcripts never hit MySQL's 64 KB ``TEXT`` cap.
All write/read paths are scoped by ``user_id`` for ownership isolation.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.roundtable_drafts.model import (
RoundtableDraftRow,
RoundtableRecommendHistoryRow,
)
_UNSET = object()
def _loads(value: Any) -> Any:
if not isinstance(value, str):
return value
if not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class RoundtableDraftRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── drafts ────────────────────────────────────────────────────────────
@staticmethod
def _draft_to_meta(row: RoundtableDraftRow) -> dict[str, Any]:
"""Lightweight projection for the history dropdown (no big blobs)."""
return {
"id": row.id,
"title": row.title,
"furthest_step": row.furthest_step,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
@classmethod
def _draft_to_dict(cls, row: RoundtableDraftRow) -> dict[str, Any]:
d = cls._draft_to_meta(row)
d["step1"] = _loads(row.step1)
d["step2"] = _loads(row.step2)
d["step3"] = _loads(row.step3)
return d
async def list_drafts(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
stmt = (
select(RoundtableDraftRow)
.where(RoundtableDraftRow.user_id == user_id)
.order_by(RoundtableDraftRow.updated_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._draft_to_meta(row) for row in result.scalars()]
async def get_draft(self, draft_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableDraftRow, draft_id)
if row is None or row.user_id != user_id:
return None
return self._draft_to_dict(row)
async def create_draft(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = RoundtableDraftRow(
id=data["id"],
user_id=user_id,
title=(data.get("title") or "").strip(),
furthest_step=int(data.get("furthest_step") or 1),
step1=_dumps(data.get("step1")),
step2=_dumps(data.get("step2")),
step3=_dumps(data.get("step3")),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._draft_to_dict(row)
async def update_draft(
self,
draft_id: str,
user_id: str,
*,
title=_UNSET,
furthest_step=_UNSET,
step1=_UNSET,
step2=_UNSET,
step3=_UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableDraftRow, draft_id)
if row is None or row.user_id != user_id:
return None
if title is not _UNSET and title is not None:
row.title = str(title).strip()
if furthest_step is not _UNSET and furthest_step is not None:
row.furthest_step = int(furthest_step)
if step1 is not _UNSET:
row.step1 = _dumps(step1)
if step2 is not _UNSET:
row.step2 = _dumps(step2)
if step3 is not _UNSET:
row.step3 = _dumps(step3)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._draft_to_dict(row)
async def delete_draft(self, draft_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(RoundtableDraftRow, draft_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
# ── recommendation history ────────────────────────────────────────────
@staticmethod
def _history_to_dict(row: RoundtableRecommendHistoryRow) -> dict[str, Any]:
return {
"id": row.id,
"draft_id": row.draft_id,
"objective": row.objective,
"status": row.status,
"model": row.model,
"rationale": row.rationale,
"picks": _loads(row.picks) or [],
"candidates": _loads(row.candidates) or [],
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
}
async def list_recommendations(
self, draft_id: str, user_id: str, *, limit: int = 20
) -> list[dict[str, Any]]:
stmt = (
select(RoundtableRecommendHistoryRow)
.where(
RoundtableRecommendHistoryRow.draft_id == draft_id,
RoundtableRecommendHistoryRow.user_id == user_id,
)
.order_by(RoundtableRecommendHistoryRow.created_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._history_to_dict(row) for row in result.scalars()]
async def add_recommendation(self, user_id: str, draft_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = RoundtableRecommendHistoryRow(
id=data["id"],
user_id=user_id,
draft_id=draft_id,
objective=(data.get("objective") or "").strip()[:512],
status=(data.get("status") or "done")[:32],
model=(data.get("model") or None),
rationale=data.get("rationale") or None,
picks=_dumps(data.get("picks") or []),
candidates=_dumps(data.get("candidates") or []),
created_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._history_to_dict(row)
@@ -7,7 +7,11 @@ import re
from typing import Any
from urllib.parse import quote
from deerflow.config.extensions_config import SkillDisplayConfig, get_extensions_config
from deerflow.config.extensions_config import (
SkillDisplayConfig,
SkillDisplayResultSetConfig,
get_extensions_config,
)
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
@@ -222,7 +226,11 @@ def adapt_reference_batches_with_displays(
)
except Exception:
continue
if not config.enabled or config.mode == "none":
has_enabled_result_set = any(
result_set.enabled and result_set.mode != "none"
for result_set in config.result_sets
)
if not config.enabled or (config.mode == "none" and not has_enabled_result_set):
continue
normalized_displays[skill_name] = config
@@ -257,13 +265,30 @@ def adapt_reference_batches_with_displays(
raw = source.get("raw")
if display is not None and isinstance(raw, dict):
index_value = int(source.get("index") or len(sources) + 1)
display_for_source: SkillDisplayConfig | SkillDisplayResultSetConfig = display
result_set_key = source.get("resultSetKey")
if result_set_key:
display_for_source = next(
(
result_set
for result_set in display.result_sets
if result_set.key == result_set_key
),
display,
)
elif display.result_sets and display.mode == "none":
sources.append(source)
continue
configured = _normalize_with_display_config(
raw,
index_value,
skill_name,
display,
display_for_source,
)
if configured is not None:
if isinstance(display_for_source, SkillDisplayResultSetConfig):
configured["resultSetKey"] = display_for_source.key
configured["resultSetLabel"] = display_for_source.label
sources.append(
{
**source,
@@ -318,6 +343,8 @@ def _source_debug_item(source: dict[str, Any]) -> dict[str, Any]:
"id": source.get("id"),
"skillName": source.get("skillName"),
"mode": source.get("mode"),
"resultSetKey": source.get("resultSetKey"),
"resultSetLabel": source.get("resultSetLabel"),
"title": source.get("title"),
"source": source.get("source"),
"author": source.get("author"),
@@ -359,6 +386,16 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
tool_contexts_by_id,
)
payload = _extract_tool_payload(message)
configured_sources = _collect_configured_sources(
payload,
len(sources) + 1,
tool_context,
)
if configured_sources is not None:
for normalized in configured_sources:
normalized["index"] = len(sources) + 1
sources.append(normalized)
continue
for raw_item in _first_list(payload):
normalized = _normalize_reference_item(
raw_item,
@@ -373,6 +410,44 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
return sources
def _collect_configured_sources(
payload: Any,
fallback_index: int,
tool_context: dict[str, Any] | None,
) -> list[dict[str, Any]] | None:
display_match = _resolve_display_match({}, payload, tool_context)
if display_match is None:
return None
skill_name, display = display_match
configs: list[SkillDisplayConfig | SkillDisplayResultSetConfig] = [
result_set
for result_set in display.result_sets
if result_set.enabled and result_set.mode != "none"
]
if not configs:
configs = [display]
sources: list[dict[str, Any]] = []
for config in configs:
raw_items = _list_at_path(payload, config.result_path)
if raw_items is None:
continue
for raw_item in raw_items:
normalized = _normalize_with_display_config(
raw_item,
fallback_index + len(sources),
skill_name,
config,
)
if normalized is None:
continue
if isinstance(config, SkillDisplayResultSetConfig):
normalized["resultSetKey"] = config.key
normalized["resultSetLabel"] = config.label
sources.append(normalized)
return sources
def _find_reference_batch_anchor(messages: list[Any], start: int, end: int) -> Any | None:
answer = _find_last_assistant_answer(messages, start, end)
if answer is not None:
@@ -622,6 +697,19 @@ def _first_list(payload: Any) -> list[Any]:
return []
def _list_at_path(payload: Any, path: str | None) -> list[dict[str, Any]] | None:
cursor = payload
if path:
for part in path.split("."):
if isinstance(cursor, dict):
cursor = cursor.get(part)
else:
return None
if not isinstance(cursor, list):
return None
return [item for item in cursor if isinstance(item, dict)]
def _nested_records(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, dict):
return []
@@ -654,7 +742,13 @@ def _enabled_skill_displays() -> dict[str, SkillDisplayConfig]:
if state.display is not None
and state.enabled
and state.display.enabled
and state.display.mode != "none"
and (
state.display.mode != "none"
or any(
result_set.enabled and result_set.mode != "none"
for result_set in state.display.result_sets
)
)
}
except Exception:
return {}
@@ -705,7 +799,7 @@ def _normalize_with_display_config(
raw: dict[str, Any],
fallback_index: int,
skill_name: str,
display: SkillDisplayConfig,
display: SkillDisplayConfig | SkillDisplayResultSetConfig,
) -> dict[str, Any] | None:
title = _stringify(_get_by_path(raw, display.title_field), f"结果 {fallback_index}")
snippet = _stringify(_get_by_path(raw, display.summary_field), "")
@@ -737,7 +831,10 @@ def _normalize_with_display_config(
}
def _resolve_display_link(raw: dict[str, Any], display: SkillDisplayConfig) -> str | None:
def _resolve_display_link(
raw: dict[str, Any],
display: SkillDisplayConfig | SkillDisplayResultSetConfig,
) -> str | None:
url = _stringify(_get_by_path(raw, display.url_field), "")
item_id = _stringify(_get_by_path(raw, display.id_field), "")
if display.link_strategy == "url":
@@ -1,8 +1,16 @@
from .registry import get_scheduled_task_service, set_scheduled_task_service
from .service import DuplicateTaskNameError, MarkdownRequirementError, ScheduledTaskService
from .service import (
DuplicateFavoriteError,
DuplicateTaskNameError,
HtmlGenerationError,
MarkdownRequirementError,
ScheduledTaskService,
)
__all__ = [
"DuplicateFavoriteError",
"DuplicateTaskNameError",
"HtmlGenerationError",
"MarkdownRequirementError",
"ScheduledTaskService",
"get_scheduled_task_service",
@@ -18,7 +18,7 @@ class ElementDeliveryClient:
self.access_token = access_token
@classmethod
def from_env(cls) -> "ElementDeliveryClient | None":
def from_env(cls) -> ElementDeliveryClient | None:
enabled = os.environ.get("ELEMENT_ENABLED", "true").strip().lower()
if enabled in {"0", "false", "no", "off"}:
return None
@@ -0,0 +1,414 @@
"""HTML page generation helpers for ``html_page`` scheduled tasks.
Pure, dependency-light utilities the scheduler uses to turn a GLM5 response
into a safe, previewable HTML document:
- :func:`extract_json_object` — fault-tolerant JSON extraction from a model
reply that may be wrapped in prose or ```json fences.
- :func:`sanitize_html` — strip script-execution vectors (``<script>``,
``<iframe>``, ``on*`` handlers, ``javascript:`` URLs, …) using BeautifulSoup.
- :func:`validate_html` — cheap structural checks (doctype/html/head/body,
non-empty body, no residual scripts) that gate the optional repair pass.
- ``build_*_prompt`` — the GLM5 prompt templates (generation / repair / style
extraction).
- :func:`screenshot_html` — best-effort Playwright screenshot, a no-op when
Playwright is not installed (the offline image ships without a browser).
Kept free of any ``app.*`` import so it stays inside the harness boundary.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any
logger = logging.getLogger(__name__)
# Tags removed wholesale — they either execute code or load remote active
# content that the iframe sandbox alone shouldn't be relied on to contain.
_DANGEROUS_TAGS = frozenset(
{"script", "iframe", "object", "embed", "applet", "frame", "frameset", "base"}
)
# URL schemes that can execute script when navigated/loaded.
_DANGEROUS_URL_RE = re.compile(r"^\s*(javascript|vbscript|data:text/html)", re.IGNORECASE)
# Attributes carrying a URL we must scheme-check.
_URL_ATTRS = frozenset({"href", "src", "action", "formaction", "xlink:href", "background", "poster"})
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
"""Split a ``---`` YAML frontmatter block from the body of a SKILL.md.
Returns ``(frontmatter_dict, body)``. When there is no frontmatter the dict
is empty and the body is the whole text.
"""
if not text or not text.lstrip().startswith("---"):
return {}, text or ""
lines = text.splitlines()
# Skip leading blank lines before the opening fence.
start = 0
while start < len(lines) and not lines[start].strip():
start += 1
end = None
for i in range(start + 1, len(lines)):
if lines[i].strip() == "---":
end = i
break
if end is None:
return {}, text
fm_block = "\n".join(lines[start + 1 : end])
body = "\n".join(lines[end + 1 :])
try:
import yaml
data = yaml.safe_load(fm_block) or {}
if not isinstance(data, dict):
data = {}
except Exception:
data = {}
return data, body
def extract_json_object(text: str | None) -> dict[str, Any] | None:
"""Best-effort parse of a JSON object from a model reply.
Handles three common shapes: a bare JSON object, a ```json fenced block,
and a JSON object embedded in surrounding prose. Returns ``None`` when no
balanced object can be recovered.
"""
if not text or not isinstance(text, str):
return None
candidate = text.strip()
# Strip a single ```json ... ``` (or ``` ... ```) fence if present.
fence = re.search(r"```(?:json)?\s*(.*?)```", candidate, flags=re.DOTALL | re.IGNORECASE)
if fence:
candidate = fence.group(1).strip()
# Fast path: the whole thing is already valid JSON.
try:
parsed = json.loads(candidate)
return parsed if isinstance(parsed, dict) else None
except (ValueError, TypeError):
pass
# Fallback: scan for the first balanced ``{ ... }`` and parse that.
start = candidate.find("{")
if start == -1:
return None
depth = 0
in_string = False
escape = False
for index in range(start, len(candidate)):
char = candidate[index]
if in_string:
if escape:
escape = False
elif char == "\\":
escape = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
blob = candidate[start : index + 1]
try:
parsed = json.loads(blob)
except (ValueError, TypeError):
return None
return parsed if isinstance(parsed, dict) else None
return None
def recover_html_document(text: str | None) -> str | None:
"""Slice a complete HTML document out of a model reply.
Used as a fallback when the JSON wrapper around the ``html`` field can't be
parsed — the model commonly emits *literal* newlines and *unescaped* quotes
(``"zh-CN"``) inside the value, which is invalid JSON, so
:func:`extract_json_object` returns ``None``. Without this, the caller would
archive the whole ``{"title": ..., "html": "<!doctype html>…"}`` blob as the
page. We locate the ``<!doctype html>`` / ``<html …>`` start and the final
``</html>`` and return just that span, dropping the surrounding JSON
scaffolding. Also accepts a reply that is already bare HTML. Returns
``None`` when no HTML document can be found.
"""
if not text or not isinstance(text, str):
return None
lowered = text.lower()
start = lowered.find("<!doctype html")
if start == -1:
start = lowered.find("<html")
if start == -1:
return None
end = lowered.rfind("</html>")
candidate = text[start:] if end == -1 else text[start : end + len("</html>")]
candidate = candidate.strip()
# If the html had been a properly-escaped JSON string value (the JSON failed
# to parse for some *other* reason), unescape the common sequences so they
# don't survive into the rendered page. Only triggers when escapes exist, so
# genuinely-literal HTML (the usual broken-JSON case) is left untouched.
if "\\n" in candidate or '\\"' in candidate or "\\/" in candidate or "\\t" in candidate:
candidate = (
candidate.replace('\\"', '"')
.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace("\\/", "/")
)
return candidate or None
def sanitize_html(html: str) -> tuple[str, list[str]]:
"""Remove script-execution vectors from ``html``.
Returns ``(cleaned_html, removed)`` where ``removed`` is a human-readable
report of what was stripped (used for logging / repair feedback). The
cleaner is defensive but not a full HTML validator — pair it with an
``iframe sandbox`` (no ``allow-scripts``) on the preview side.
"""
if not html or not isinstance(html, str):
return "", []
try:
from bs4 import BeautifulSoup
except Exception: # pragma: no cover - bs4 is a declared dependency
logger.warning("BeautifulSoup unavailable; returning HTML unsanitized")
return html, []
removed: list[str] = []
soup = BeautifulSoup(html, "html.parser")
# 1) Drop dangerous tags entirely.
for tag in soup.find_all(True):
name = (tag.name or "").lower()
if name in _DANGEROUS_TAGS:
removed.append(f"<{name}>")
tag.decompose()
continue
# External stylesheet / preload links pull remote resources.
if name == "link":
href = str(tag.get("href") or "")
rel = " ".join(tag.get("rel") or []) if isinstance(tag.get("rel"), list) else str(tag.get("rel") or "")
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
removed.append("<link stylesheet>")
tag.decompose()
continue
# meta refresh can redirect to arbitrary URLs.
if name == "meta" and str(tag.get("http-equiv") or "").lower() == "refresh":
removed.append("<meta refresh>")
tag.decompose()
continue
# 2) Strip event handlers and dangerous URL schemes from surviving tags.
for tag in soup.find_all(True):
for attr in list(tag.attrs.keys()):
lowered = attr.lower()
if lowered.startswith("on"):
removed.append(f"{lowered}=")
del tag.attrs[attr]
continue
if lowered in _URL_ATTRS:
value = tag.attrs[attr]
value_str = value if isinstance(value, str) else " ".join(value)
if _DANGEROUS_URL_RE.match(value_str or ""):
removed.append(f"{lowered}={value_str[:24]}")
del tag.attrs[attr]
return str(soup), removed
def validate_html(html: str) -> list[str]:
"""Return a list of structural problems with ``html`` (empty == valid).
A page only counts as a complete document when it has the full skeleton
(doctype/html/head/body), a non-empty ``<title>``, visible body content
(no white-screen), no residual scripts, and no external stylesheet links.
"""
errors: list[str] = []
if not html or not isinstance(html, str):
return ["empty html"]
lowered = html.lower()
if "<!doctype html" not in lowered:
errors.append("缺少 <!doctype html> 声明")
if "<html" not in lowered:
errors.append("缺少 <html> 根元素")
if "<head" not in lowered:
errors.append("缺少 <head> 元素")
if "<body" not in lowered:
errors.append("缺少 <body> 元素")
if "<script" in lowered:
errors.append("仍包含 <script> 标签(不允许脚本)")
try:
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
# Title must exist and be non-empty.
title_el = soup.find("title")
if title_el is None or not title_el.get_text(strip=True):
errors.append("缺少非空 <title> 标题")
# External stylesheet links pull remote resources — not allowed offline.
for link in soup.find_all("link"):
href = str(link.get("href") or "")
rel = " ".join(link.get("rel") or []) if isinstance(link.get("rel"), list) else str(link.get("rel") or "")
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
errors.append("包含外链样式表(不允许外部 CDN/资源)")
break
# Body must carry some visible content — guards against white-screen.
body = soup.body or soup
text = body.get_text(strip=True) if body else ""
has_media = bool(body.find(["img", "svg", "canvas", "table", "ul", "ol"])) if body else False
if len(text) < 5 and not has_media:
errors.append("页面正文为空,可能白屏")
except Exception: # pragma: no cover
pass
return errors
# ---------------------------------------------------------------------------
# Prompt templates
# ---------------------------------------------------------------------------
_GENERATION_TEMPLATE = """你是一个内网页面生成助手。请根据输入数据生成一个完整的 HTML 页面。
用户页面要求:
{page_prompt}
{constraint_block}本次数据:
{data}
参考页面风格:
{reference_style}
参考页面布局:
{reference_layout}
{skill_instructions}
要求:
1. 只输出 JSON,不要输出 Markdown,不要输出多余解释。
2. JSON 字段必须包含 title、html、style_summary、layout_summary。
3. html 字段的值必须是合法 JSON 字符串:其中所有换行必须写成 \\n、所有双引号必须转义为 \\",不要在字符串里直接换行或使用未转义的引号。
4. html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
5. 所有 CSS 写在 <style> 标签内,不要引用外部 CDN 或外链样式表。
6. 不要包含 <script>、外链脚本、iframe、表单提交、on* 事件属性。
7. 参考页面只用于风格和布局,页面内容必须来自“本次数据”,不要复用参考页旧数据。
8. 页面使用中文,排版适合在浏览器中直接预览。
"""
_REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问题,请修复后重新输出完整页面。
原始页面要求:
{requirements}
发现的问题:
{errors}
待修复 HTML:
{html}
要求:
1. 只输出 JSON,字段包含 fixed_html、changes(修复说明数组)。
2. fixed_html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
3. 移除所有 <script>、iframe、外链脚本、on* 事件属性、表单提交。
4. 保留原有内容与数据,只修复结构与安全问题。
"""
_STYLE_EXTRACT_TEMPLATE = """你是一个页面风格分析助手。请分析下面的 HTML 页面,提取可复用的风格与布局特征。
HTML 页面:
{html}
要求:
1. 只输出 JSON,字段包含 style_summary、layout_summary、content_structure_summary、reuse_prompt。
2. style_summary:颜色、字体、间距、圆角、阴影、视觉密度等视觉风格。
3. layout_summary:页面分区、卡片、表格、图表、导航等布局结构。
4. content_structure_summary:信息组织顺序与层级。
5. reuse_prompt:一段可直接拼接到下次生成 prompt 的中文风格说明,不要包含本页的具体数据。
"""
def _clip(value: Any, limit: int) -> str:
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
text = text or ""
return text if len(text) <= limit else text[:limit] + "\n…(内容过长已截断)"
def build_generation_prompt(
*,
page_prompt: str,
negative_prompt: str | None,
data: Any,
reference_style: str | None,
reference_layout: str | None,
skill_instructions: str | None = None,
data_limit: int = 16000,
skill_limit: int = 30000,
) -> str:
skill_block = ""
if skill_instructions and skill_instructions.strip():
skill_block = "\n可用页面生成技能规范(请严格遵循):\n" + _clip(skill_instructions.strip(), skill_limit) + "\n"
# The constraint block is only emitted when the user actually supplied one,
# and is framed as a top-priority hard rule that overrides the page
# requirements / reference page / data on conflict — otherwise the model
# tends to treat "限制条件" as a soft hint and ignore it.
constraint_block = ""
neg = (negative_prompt or "").strip()
if neg:
constraint_block = (
"\n限制条件(最高优先级的硬性约束,必须逐条严格遵守;"
"若与上面的页面要求、参考页面或本次数据发生冲突,一律以这里的限制条件为准):\n"
+ neg
+ "\n\n"
)
return _GENERATION_TEMPLATE.format(
page_prompt=(page_prompt or "(未提供,使用通用看板风格)").strip(),
constraint_block=constraint_block,
data=_clip(data, data_limit),
reference_style=(reference_style or "无").strip(),
reference_layout=(reference_layout or "无").strip(),
skill_instructions=skill_block,
)
def build_repair_prompt(*, html: str, errors: list[str], requirements: str | None) -> str:
return _REPAIR_TEMPLATE.format(
requirements=(requirements or "无").strip(),
errors="\n".join(f"- {e}" for e in errors) or "- 未知问题",
html=_clip(html, 32000),
)
def build_style_extract_prompt(*, html: str) -> str:
return _STYLE_EXTRACT_TEMPLATE.format(html=_clip(html, 32000))
async def screenshot_html(html: str, output_path) -> str | None:
"""Best-effort PNG screenshot of ``html`` for white-screen / console QA.
Returns the screenshot path on success, or ``None`` when Playwright (or a
usable browser) is unavailable — the offline image ships without one, so
this must degrade silently rather than fail the run.
"""
try:
from playwright.async_api import async_playwright
except Exception:
return None
try:
async with async_playwright() as pw:
browser = await pw.chromium.launch()
try:
page = await browser.new_page()
await page.set_content(html, wait_until="networkidle")
await page.screenshot(path=str(output_path), full_page=True)
finally:
await browser.close()
return str(output_path)
except Exception:
logger.debug("Playwright screenshot failed; skipping QA screenshot", exc_info=True)
return None
@@ -22,6 +22,7 @@ from deerflow.config.agents_config import AGENT_ID_PATTERN
from deerflow.config.paths import get_paths
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.scheduler import html_page as html_page_utils
from deerflow.runtime.scheduler.element import ElementDeliveryClient
from deerflow.runtime.user_context import reset_current_user, set_current_user
from deerflow.utils.time import now_iso
@@ -36,6 +37,11 @@ _RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
# Max HTML length (chars) inlined into the run's ``result_json``. Beyond this
# the inline copy is dropped to stay under MySQL's 64 KB ``TEXT`` cap; the full
# document is always retrievable from the page store (LONGTEXT) + index.html.
_MAX_INLINE_RESULT_HTML = 60000
# Minimum length (in characters) the generated Markdown file's content must
# reach when a task has ``require_markdown`` enabled. The model is NOT told about
# this threshold — it is enforced silently by the service.
@@ -58,6 +64,22 @@ class MarkdownRequirementError(Exception):
"""
class HtmlGenerationError(Exception):
"""Raised when an ``html_page`` run did not produce a complete, valid HTML
document (missing doctype/html/head/body, empty body, or residual scripts
after the sanitize + repair passes). The run is marked failed — a page only
counts as successful when it is a full, previewable HTML document."""
class DuplicateFavoriteError(Exception):
"""A user already bookmarked this page. Surfaced as HTTP 409 by the router
so the same page cannot be favorited twice."""
def __init__(self, page_id: str) -> None:
self.page_id = page_id
super().__init__(f"Page {page_id!r} is already favorited")
class DuplicateTaskNameError(Exception):
"""A user already owns a scheduled task with the requested name.
@@ -835,7 +857,7 @@ class ScheduledTaskService:
runtime_thread_id = str(uuid.uuid4())
try:
result = await asyncio.wait_for(
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
self._run_task_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
)
except TimeoutError:
@@ -875,6 +897,342 @@ class ScheduledTaskService:
await self._deliver_element_result(task, result["content"])
return
async def _run_task_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Dispatch a single execution attempt by task kind.
``html_page`` tasks render an HTML document via GLM5; everything else
runs the regular lead-agent attempt. The kind lives in
``execution_context.task_kind`` so no schema migration is needed.
"""
ctx = task.get("execution_context")
kind = ctx.get("task_kind") if isinstance(ctx, dict) else None
if kind == "html_page":
return await self._run_html_page_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
return await self._run_agent_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
@staticmethod
def _html_config(task: dict[str, Any]) -> dict[str, Any]:
ctx = task.get("execution_context")
cfg = ctx.get("html_config") if isinstance(ctx, dict) else None
return cfg if isinstance(cfg, dict) else {}
async def _invoke_chat_model(self, model_name: str | None, prompt: str) -> str:
"""Invoke a chat model once and return its text content.
Used for the GLM5 HTML generation / repair / style-extraction passes.
Thinking is disabled to keep structured-JSON output stable.
"""
from langchain_core.messages import HumanMessage
from deerflow.models import create_chat_model
# No hardcoded model: use exactly the selected model; only when none was
# selected do we fall back to the system default (config models[0]).
model = create_chat_model(name=model_name or None, thinking_enabled=False)
response = await model.ainvoke([HumanMessage(content=prompt)])
content = getattr(response, "content", response)
if isinstance(content, str):
return content
if isinstance(content, list):
parts = [part.get("text", "") if isinstance(part, dict) else str(part) for part in content]
return "\n".join(p for p in parts if p)
return str(content)
async def _gather_html_data(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> str:
"""Run the lead agent to collect source data for the HTML page.
Reuses the regular agent-run primitives (with the task's search query
folded into the prompt) and returns the agent's final text, which feeds
the GLM5 render step as ``本次数据``. Returns an empty string if the
agent produced nothing.
"""
cfg = self._html_config(task)
page_prompt = str(cfg.get("page_prompt") or "").strip()
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
gather_instruction = (
"请运用你自身的检索/搜索等技能,围绕下述页面需求获取最新、真实的资料,"
"并整理成结构化的中文要点(可用列表/小标题),供后续生成 HTML 页面使用,本步骤不要生成 HTML:\n"
)
parts = [gather_instruction]
if page_prompt:
parts.append(f"页面需求:{page_prompt}")
base_prompt = str(task.get("prompt") or "").strip()
if base_prompt:
parts.append(f"补充说明:{base_prompt}")
# Apply the user's constraints to data gathering too, not just rendering
# — a "限制条件" like "只要最近 24 小时的新闻" or "排除某主题" must shape
# which sources/data the agent collects, otherwise it silently leaks in.
if negative_prompt:
parts.append(f"限制条件(最高优先级,必须严格遵守):{negative_prompt}")
gather_task = {**task, "prompt": "\n".join(parts)}
# Never apply the markdown gate to the gather pass.
gather_ctx = dict(task.get("execution_context") or {})
gather_ctx.pop("require_markdown", None)
gather_ctx.pop("task_kind", None)
gather_ctx.pop("html_config", None)
# Use the task's selected model for the data-gathering agent too, so the
# whole HTML pipeline honors ``html_config.model_name`` instead of
# silently falling back to the default model for this step.
gather_model = str(cfg.get("model_name") or "").strip()
if gather_model:
gather_ctx["model_name"] = gather_model
gather_task["execution_context"] = gather_ctx
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
record = await self._start_agent_run(gather_task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
text = await self._extract_latest_ai_text(runtime_thread_id)
return text or ""
def _collect_html_skill_instructions(self) -> str:
"""Concatenate the bodies of enabled skills marked for HTML pages.
A skill opts in via ``metadata.kind: html_page`` in its SKILL.md
frontmatter. Only enabled skills (per the skills settings the user
toggles) are injected, so the render step honors the user's skill
selection. Best-effort — any failure yields an empty string.
"""
try:
from deerflow.skills import get_or_new_skill_storage
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
except Exception:
logger.debug("Failed to load skills for HTML render injection", exc_info=True)
return ""
blocks: list[str] = []
for skill in skills:
try:
text = Path(skill.skill_file).read_text(encoding="utf-8")
except Exception:
continue
frontmatter, body = html_page_utils.parse_frontmatter(text)
meta = frontmatter.get("metadata") if isinstance(frontmatter, dict) else None
if not (isinstance(meta, dict) and str(meta.get("kind")) == "html_page"):
continue
body = body.strip()
if body:
blocks.append(f"### 技能:{skill.name}\n{body}")
return "\n\n".join(blocks)
def _archive_html_index(self, run_id: str, html: str) -> Path | None:
"""Write the generated HTML to the run's archive dir as ``index.html``."""
try:
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id / "files"
archive_dir.mkdir(parents=True, exist_ok=True)
target = archive_dir / "index.html"
target.write_text(html, encoding="utf-8")
return target
except Exception:
logger.warning("Failed to archive generated HTML for run %s", run_id, exc_info=True)
return None
async def _run_html_page_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Generate an HTML page for an ``html_page`` task.
Pipeline: gather data via the lead agent → optionally load a reference
favorite's style/layout → GLM5 render → sanitize → validate → (one
GLM5 repair pass if invalid) → best-effort screenshot → archive
``index.html`` → persist a ``scheduled_html_pages`` row → build the run
result payload (``format: "html"`` with the full HTML inline).
"""
cfg = self._html_config(task)
# The user's selected model only — no hardcoded GLM-5. Empty → None →
# system default (config models[0]).
model_name = str(cfg.get("model_name") or "").strip() or None
page_prompt = str(cfg.get("page_prompt") or "").strip()
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
reference_favorite_id = cfg.get("reference_favorite_id") or None
# 1) Gather source data (real, via the agent's search tools).
data_text = await self._gather_html_data(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
# 2) Load reference style/layout from a favorite if one was selected.
reference_style = None
reference_layout = None
if reference_favorite_id:
store = self._html_store
if store is not None:
try:
favorite = await store.get_favorite(str(reference_favorite_id))
except Exception:
favorite = None
if favorite is not None:
reference_style = favorite.get("reuse_prompt") or favorite.get("style_summary")
reference_layout = favorite.get("layout_summary")
# 3) Render with the chosen model, injecting any enabled HTML skills.
skill_instructions = self._collect_html_skill_instructions()
gen_prompt = html_page_utils.build_generation_prompt(
page_prompt=page_prompt,
negative_prompt=negative_prompt,
data=data_text,
reference_style=reference_style,
reference_layout=reference_layout,
skill_instructions=skill_instructions,
)
raw = await self._invoke_chat_model(model_name, gen_prompt)
parsed = html_page_utils.extract_json_object(raw) or {}
title = str(parsed.get("title") or task.get("name") or "生成页面").strip()
html = parsed.get("html")
if not isinstance(html, str) or "<" not in html:
# The JSON wrapper couldn't be parsed (the model emitted literal
# newlines / unescaped quotes inside the "html" value), or it replied
# with bare HTML. Either way, recover just the HTML document so we
# never archive the {"title":...,"html":"..."} JSON blob as the page.
recovered = html_page_utils.recover_html_document(raw)
if recovered is None:
raise ValueError("模型未返回有效的 HTML(JSON 解析失败)")
html = recovered
style_summary = str(parsed.get("style_summary") or "").strip() or None
layout_summary = str(parsed.get("layout_summary") or "").strip() or None
# 4) Sanitize + validate, with one repair pass on failure.
html, _removed = html_page_utils.sanitize_html(html)
errors = html_page_utils.validate_html(html)
if errors:
logger.info("Generated HTML failed validation (%s); attempting repair", errors)
repair_prompt = html_page_utils.build_repair_prompt(
html=html, errors=errors, requirements=page_prompt or task.get("prompt")
)
try:
repaired_raw = await self._invoke_chat_model(model_name, repair_prompt)
repaired = html_page_utils.extract_json_object(repaired_raw) or {}
fixed = repaired.get("fixed_html")
if isinstance(fixed, str) and "<" in fixed:
fixed_clean, _ = html_page_utils.sanitize_html(fixed)
if not html_page_utils.validate_html(fixed_clean):
html = fixed_clean
except Exception:
logger.warning("HTML repair pass failed; keeping sanitized original", exc_info=True)
# 4b) Completeness gate — a page only counts as successful when it is a
# full, valid HTML document. If it still fails validation after the
# repair pass, fail the run (no half-baked page is archived/persisted).
final_errors = html_page_utils.validate_html(html)
if final_errors:
raise HtmlGenerationError(
"HTML 生成未通过完整性校验:" + ";".join(final_errors) + "。请调整页面要求或重试。"
)
# 5) Best-effort screenshot QA (no-op without Playwright).
screenshot_path = None
try:
shot_target = get_paths().base_dir / "scheduled-task-runs" / run_id / "files" / "preview.png"
shot_target.parent.mkdir(parents=True, exist_ok=True)
screenshot_path = await html_page_utils.screenshot_html(html, shot_target)
except Exception:
screenshot_path = None
# 6) Archive index.html.
archive_path = self._archive_html_index(run_id, html)
files: list[dict[str, Any]] = []
if archive_path is not None:
files.append(
{
"name": "index.html",
"path": str(archive_path),
"virtual_path": "/mnt/user-data/outputs/index.html",
"mime_type": "text/html",
"size": archive_path.stat().st_size if archive_path.exists() else None,
"kind": "html",
}
)
if screenshot_path:
shot = Path(screenshot_path)
files.append(
{
"name": "preview.png",
"path": str(shot),
"mime_type": "image/png",
"size": shot.stat().st_size if shot.exists() else None,
"kind": "artifact",
}
)
data_snapshot = {"data_text": data_text[:8000] if isinstance(data_text, str) else ""}
prompt_snapshot = {
"page_prompt": page_prompt,
"negative_prompt": negative_prompt,
"reference_favorite_id": reference_favorite_id,
"model_name": model_name,
}
# 7) Persist the page asset (best-effort — failure must not fail the run).
store = self._html_store
if store is not None:
try:
await store.create_page(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"run_id": run_id,
"user_id": task["user_id"],
"title": title,
"html_content": html,
"html_file_path": str(archive_path) if archive_path else None,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"reference_favorite_id": reference_favorite_id,
"style_summary": style_summary,
"layout_summary": layout_summary,
}
)
except Exception:
logger.warning("Failed to persist scheduled_html_pages row for run %s", run_id, exc_info=True)
task_tz = ZoneInfo(task.get("timezone") or "UTC")
local_time = scheduled_for.astimezone(task_tz)
# ``scheduled_task_runs.result_json`` is TEXT (64 KB) on MySQL, so only
# inline the HTML when it comfortably fits. The full document always
# lives in ``scheduled_html_pages.html_content`` (LONGTEXT) + the
# archived ``index.html``; the ``/runs/{id}/html`` endpoints read from
# there, so previews work even when the inline copy is omitted here.
inline_html = html if len(html) <= _MAX_INLINE_RESULT_HTML else None
return {
"title": f"{title} - {local_time.strftime('%Y-%m-%d %H:%M')}",
"content": page_prompt or f"HTML 页面已生成:{title}",
"format": "html",
"html": inline_html,
"style_summary": style_summary,
"layout_summary": layout_summary,
"data_snapshot": data_snapshot,
"prompt_snapshot": prompt_snapshot,
"files": files,
"created_at": datetime.now(UTC).isoformat(),
}
async def _run_agent_attempt(
self,
task: dict[str, Any],
@@ -1275,3 +1633,132 @@ class ScheduledTaskService:
async def unsubscribe_task(self, user_id: str, task_id: str) -> bool:
return await self.store.delete_subscription(task_id, user_id)
# --- HTML pages / favorites ---
@property
def _html_store(self):
state = getattr(self.app, "state", None)
return getattr(state, "html_page_store", None)
async def get_html_page_for_run(self, run_id: str) -> dict[str, Any] | None:
"""Return the generated HTML page for a run (page-store row or run result)."""
store = self._html_store
if store is not None:
page = await store.get_page_by_run(run_id)
if page is not None:
return page
# Fallback to the inline HTML on the run result.
run = await self.store.get_run_any(run_id)
if run is None:
return None
result = run.get("result") if isinstance(run.get("result"), dict) else None
if not result or not isinstance(result.get("html"), str):
return None
return {
"run_id": run_id,
"task_id": run.get("task_id"),
"title": result.get("title"),
"html_content": result.get("html"),
"style_summary": result.get("style_summary"),
"layout_summary": result.get("layout_summary"),
"created_at": result.get("created_at"),
}
async def get_latest_html_page_for_task(self, task_id: str) -> dict[str, Any] | None:
store = self._html_store
if store is not None:
page = await store.latest_page_for_task(task_id)
if page is not None:
return page
# Fallback: newest run that carries inline HTML.
runs = await self.store.list_runs_any(task_id, limit=20)
for run in runs:
result = run.get("result") if isinstance(run.get("result"), dict) else None
if result and isinstance(result.get("html"), str):
return await self.get_html_page_for_run(run["id"])
return None
async def list_html_page_favorites(self, user_id: str) -> list[dict[str, Any]]:
store = self._html_store
if store is None:
return []
return await store.list_favorites(user_id)
async def get_html_page_favorite(self, user_id: str, favorite_id: str) -> dict[str, Any] | None:
store = self._html_store
if store is None:
return None
favorite = await store.get_favorite(favorite_id)
if favorite is None or favorite.get("user_id") != user_id:
return None
return favorite
async def delete_html_page_favorite(self, user_id: str, favorite_id: str) -> bool:
store = self._html_store
if store is None:
return False
return await store.delete_favorite(favorite_id, user_id)
async def create_html_page_favorite(
self,
user_id: str,
page_id: str,
*,
title: str | None = None,
description: str | None = None,
tags: list[str] | None = None,
) -> dict[str, Any] | None:
"""Bookmark a generated page, extracting a reusable style summary via GLM5.
Returns ``None`` when the page does not exist or is not owned by the
caller. Style extraction is best-effort — a failure still produces a
favorite (without the reuse hints) so the user's bookmark is never lost.
"""
store = self._html_store
if store is None:
return None
page = await store.get_page(page_id)
if page is None or page.get("user_id") != user_id:
return None
# A page can only be bookmarked once per user.
existing = await store.get_favorite_by_page(user_id, page_id)
if existing is not None:
raise DuplicateFavoriteError(page_id)
html = page.get("html_content") or ""
style_summary = page.get("style_summary")
layout_summary = page.get("layout_summary")
content_structure_summary = None
reuse_prompt = None
if html:
try:
# Reuse the model the page was generated with (recorded in the
# page's prompt snapshot) so favoriting doesn't fall back to the
# default model.
extract_model = (page.get("prompt_snapshot") or {}).get("model_name")
extract_prompt = html_page_utils.build_style_extract_prompt(html=html)
raw = await self._invoke_chat_model(extract_model, extract_prompt)
extracted = html_page_utils.extract_json_object(raw) or {}
style_summary = extracted.get("style_summary") or style_summary
layout_summary = extracted.get("layout_summary") or layout_summary
content_structure_summary = extracted.get("content_structure_summary")
reuse_prompt = extracted.get("reuse_prompt")
except Exception:
logger.warning("HTML style extraction failed for page %s; saving favorite without hints", page_id, exc_info=True)
return await store.create_favorite(
{
"id": str(uuid.uuid4()),
"user_id": user_id,
"page_id": page_id,
"title": (title or page.get("title") or "收藏页面").strip(),
"description": (description or None),
"tags": tags or [],
"html_snapshot": html,
"style_summary": style_summary,
"layout_summary": layout_summary,
"content_structure_summary": content_structure_summary,
"reuse_prompt": reuse_prompt,
}
)
@@ -6,11 +6,13 @@ from .present_file_tool import present_file_tool
from .router_quality_tool import router_quality_check
from .scheduled_task_tool import scheduled_task_tool
from .setup_agent_tool import setup_agent
from .setup_writing_tool import setup_writing
from .task_tool import task_tool
from .view_image_tool import view_image_tool
__all__ = [
"setup_agent",
"setup_writing",
"present_file_tool",
"ask_clarification_tool",
"agent_orchestration_tool",
@@ -56,7 +56,7 @@ def notebook_search_tool(query: str, top_k: int = 10) -> str:
headers={"Authorization": f"Bearer {token}"},
json={
"query": query,
"source_ids": [source_ids],
"source_ids": source_ids,
"search_space_id": search_space_id,
"top_k": top_k,
},
@@ -27,7 +27,7 @@ def setup_agent(
description: One-line description of what the agent does.
name: Optional human-readable display name for the agent. If omitted,
the agent's stable id is used as the display name.
skills: Optional list of skill names this agent should use. None or empty means no skills.
skills: Optional list of skill names this agent should use. None or empty means no skills. Maximum 5 skills allowed.
"""
agent_name: str | None = (runtime.context.get("agent_id") or runtime.context.get("agent_name")) if runtime.context else None
@@ -48,7 +48,7 @@ def setup_agent(
if description:
config_data["description"] = description
if skills is not None:
config_data["skills"] = skills
config_data["skills"] = skills[:5]
config_file = agent_dir / "config.yaml"
with open(config_file, "w", encoding="utf-8") as f:
@@ -0,0 +1,53 @@
"""写作配置助手用的提交工具。
与 ``setup_agent`` 同构:模型在「写作配置 Q&A」里与用户聊清需求后,调用本工具把建议的
写作配置「提交」出来。实际它总是被 ``SetupWritingApprovalMiddleware`` 在 ``after_model``
拦截(不真正执行)——拦截后把建议字段打包进审批 ToolMessage,前端据此回填左侧写作表单。
因此本工具的函数体基本是兜底:万一未被拦截(如审批被预先放行),也只回一句确认、不落任何盘。
"""
from __future__ import annotations
import logging
from langchain_core.messages import ToolMessage
from langchain_core.tools import tool
from langgraph.prebuilt import ToolRuntime
from langgraph.types import Command
logger = logging.getLogger(__name__)
@tool
def setup_writing(
user_intent: str,
runtime: ToolRuntime,
article_type: str | None = None,
word_count_target: int | None = None,
target_audience: str | None = None,
writing_mode: str | None = None,
material_source: str | None = None,
outline: str | None = None,
) -> Command:
"""把与用户聊清的写作配置提交给前端表单(不直接开始写作)。
Args:
user_intent: 用户的写作意图(精炼成一句话的写作目标)。
article_type: 文章类型的 key(须是系统给定的合法类型之一;不确定可留空)。
word_count_target: 目标字数(正整数)。
target_audience: 目标读者(如「行业专家」「普通读者」)。
writing_mode: 写作模式,'strict'(严格依据素材)或 'loose'(可结合通用知识)。
material_source: 素材来源,'knowledge_base'(知识库)或 'skill'(技能检索)。
outline: 若用户已提供大纲,原样填入其大纲文本(保留其层级与标题),否则留空。
"""
logger.info("[setup_writing] proposed writing config submitted (intercepted by approval middleware)")
return Command(
update={
"messages": [
ToolMessage(
content="写作配置已生成,请在右侧确认后应用到表单。",
tool_call_id=runtime.tool_call_id,
)
]
}
)
@@ -51,20 +51,33 @@ def view_image_tool(
runtime: ToolRuntime[ContextT, ThreadState],
image_path: str,
tool_call_id: Annotated[str, InjectedToolCallId],
question: str | None = None,
) -> Command:
"""Read an image file.
"""Read and understand an image file.
Use this tool to read an image file and make it available for display.
This is the ONLY correct way to understand an image. The image is analyzed by
the configured vision model and its description/answer is returned to you as
the tool result. Treat that result as authoritative.
When to use the view_image tool:
- When you need to view an image file.
- Whenever you need to view, describe, or answer a question about an image file.
When NOT to use the view_image tool:
- For non-image files (use present_files instead)
- For multiple files at once (use present_files instead)
IMPORTANT — do NOT attempt to understand images any other way:
- Never read, open, decode, or parse an image with read_file, grep, bash,
python, PIL/Pillow, OpenCV, EXIF parsing, or byte/hex inspection. Those
cannot interpret visual content and will only waste turns.
- If `view_image` returns a weak or incomplete description, do not fall back
to code. Either call `view_image` again with a more specific `question`,
or tell the user the configured vision model could not extract more detail
(the vision model may need to be changed in config).
Args:
image_path: Absolute /mnt/user-data virtual path to the image file. Common formats supported: jpg, jpeg, png, webp.
question: Optional specific question to ask about the image (e.g. "What is the total on this invoice?"). When omitted, a general detailed description is produced.
"""
from deerflow.sandbox.exceptions import SandboxRuntimeError
from deerflow.sandbox.tools import (
@@ -153,8 +166,27 @@ def view_image_tool(
mime_type = detected_mime_type
image_base64 = base64.b64encode(image_data).decode("utf-8")
# Update viewed_images in state
# The merge_viewed_images reducer will handle merging with existing images
# Delegated recognition: when a dedicated vision model is configured, send
# the image to that model and return its description as the tool result, so
# a text-only main chat model can still "see" the image.
from deerflow.models.vision import is_delegated_vision, recognize_image
if is_delegated_vision():
try:
description = recognize_image(image_base64, mime_type, question=question)
except Exception as e: # noqa: BLE001 - surface as a recoverable tool error
return Command(
update={"messages": [ToolMessage(f"Error recognizing image: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
)
if not description:
description = "The vision model returned an empty description."
return Command(
update={"messages": [ToolMessage(f"Image recognition result for {image_path}:\n\n{description}", tool_call_id=tool_call_id)]},
)
# Native vision: store the image so ViewImageMiddleware injects it into the
# (vision-capable) main model before the next LLM call.
# The merge_viewed_images reducer will handle merging with existing images.
new_viewed_images = {image_path: {"base64": image_base64, "mime_type": mime_type}}
return Command(
@@ -142,11 +142,14 @@ def get_available_tools(
if model_name is None and config.models:
model_name = config.models[0].name
# Add view_image_tool only if the model supports vision
model_config = config.get_model_config(model_name) if model_name else None
if model_config is not None and model_config.supports_vision:
# Add view_image_tool when vision is available — either the main model
# supports vision natively, or a dedicated vision model is configured
# (vision.model_name) so recognition can be delegated to it.
from deerflow.models.vision import vision_enabled
if vision_enabled(model_name, app_config=config):
builtin_tools.append(view_image_tool)
logger.info(f"Including view_image_tool for model '{model_name}' (supports_vision=True)")
logger.info(f"Including view_image_tool for model '{model_name}' (vision available)")
# Get cached MCP tools if enabled
# NOTE: We use ExtensionsConfig.from_file() instead of config.extensions
@@ -138,12 +138,21 @@ def open_upload_file_no_symlink(base_dir: Path, filename: str) -> tuple[Path, ob
validate_path_traversal(dest, base_dir)
if not hasattr(os, "O_NOFOLLOW"):
raise UnsafeUploadPathError("Upload writes require O_NOFOLLOW support")
flags = os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW
# ``O_NOFOLLOW`` closes the TOCTOU window between the ``lstat`` above and the
# ``open`` below, but it is a POSIX-only flag — it does not exist on Windows
# (``os.O_NOFOLLOW`` is undefined there). Windows also lacks ``O_NONBLOCK``
# and needs ``O_BINARY`` to avoid CRLF translation on the raw fd. Where
# ``O_NOFOLLOW`` is unavailable we still reject pre-existing symlinks via the
# ``lstat`` check and re-verify the opened file with ``fstat`` below; the
# residual race only matters under a hostile local user, which is out of
# scope for the single-user local deployment this path targets.
flags = os.O_WRONLY | os.O_CREAT
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
if hasattr(os, "O_NONBLOCK"):
flags |= os.O_NONBLOCK
if hasattr(os, "O_BINARY"):
flags |= os.O_BINARY
try:
fd = os.open(dest, flags, 0o600)