Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xhy
This commit is contained in:
commit
3e2182acc5
275 files changed
+29592
-11370
No files matched your search
+37
@@ -74,6 +74,28 @@ def get_model(config: RunnableConfig, model_name: str | None = None):
|
||||
return create_chat_model(name=name)
|
||||
|
||||
|
||||
def get_writing_model(config: RunnableConfig, state: dict):
|
||||
"""大纲 / 写作 / 审核专用:严格使用用户在表单选定的模型,**不回退默认**。
|
||||
|
||||
用户模型在 intent_parser(首次 invocation,configurable.model_name 可用)阶段
|
||||
捕获进 ``state['model_name']``,经 checkpoint 持久化。这三个节点都在
|
||||
「确认素材 / 大纲」resume 之后才运行,而 resume 是新 invocation、其 config 不再
|
||||
带 model_name —— 所以必须从 state 读,不能从 configurable 读(否则回退到 config
|
||||
列表第一个模型,常是推理模型,既影响质量也会让 reasoning_content 走不出流式)。
|
||||
|
||||
缺失视为配置错误直接抛 ValueError —— 由各节点的 try/except 兜底成 status:error,
|
||||
而不是静默回退默认模型。
|
||||
"""
|
||||
name = ((state or {}).get("model_name") or "").strip()
|
||||
if not name:
|
||||
# 兜底再看一眼 configurable(首次同 invocation 内直跑到这里的极端情况)
|
||||
name = (config.get("configurable", {}).get("model_name", "") or "").strip()
|
||||
if not name:
|
||||
raise ValueError("未指定写作模型(model_name 为空):请在写作表单选择模型后重试")
|
||||
from deerflow.models import create_chat_model
|
||||
return create_chat_model(name=name)
|
||||
|
||||
|
||||
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
|
||||
_FULLWIDTH_FIX = {
|
||||
"“": '"', # 左双引号 “
|
||||
@@ -175,6 +197,21 @@ def parse_json(text: str, fallback: dict) -> dict:
|
||||
return fallback
|
||||
|
||||
|
||||
def reasoning_delta(chunk) -> str:
|
||||
"""提取流式 chunk 的增量「推理内容」。
|
||||
|
||||
推理模型(deepseek-reasoner / GLM 等)会把思考过程放在
|
||||
``additional_kwargs.reasoning_content`` 增量字段里,而 ``content`` 为空 ——
|
||||
只读 content 的流式循环会把这些 chunk 全部跳过,前端就看不到任何实时输出。
|
||||
各写作节点用本函数把推理增量当作 ``is_thinking=True`` 推给前端的思考框。
|
||||
|
||||
兼容不同 provider 的字段名:``reasoning_content``(主流)/ ``reasoning``。
|
||||
"""
|
||||
ak = getattr(chunk, "additional_kwargs", None) or {}
|
||||
val = ak.get("reasoning_content") or ak.get("reasoning") or ""
|
||||
return val if isinstance(val, str) else ""
|
||||
|
||||
|
||||
def _content_to_text(content) -> str:
|
||||
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
|
||||
if isinstance(content, str):
|
||||
|
||||
+6
-2
@@ -11,7 +11,7 @@ from ..state import AIWritingState, ReviewResult, ReviewIssue
|
||||
from ..prompts.editor_prompts import (
|
||||
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
|
||||
)
|
||||
from ._utils import get_ai_writing_config, get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
|
||||
from ._utils import get_ai_writing_config, get_writing_model, parse_json, _content_to_text, reasoning_delta, ThinkTagParser, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -49,6 +49,10 @@ async def _stream_one_review(
|
||||
SystemMessage(content=EDITOR_SYSTEM),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
writer({"type": "review_chunk", "category": category, "text": reasoning, "is_thinking": True})
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
@@ -118,7 +122,7 @@ async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
model = get_writing_model(config, state)
|
||||
except Exception as e:
|
||||
logger.error("editor model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
|
||||
+11
@@ -14,6 +14,7 @@ from ._utils import (
|
||||
_content_to_text,
|
||||
get_model,
|
||||
parse_json,
|
||||
reasoning_delta,
|
||||
safe_get_stream_writer,
|
||||
)
|
||||
|
||||
@@ -215,6 +216,10 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
|
||||
async for chunk in model.astream(
|
||||
[SystemMessage(content=system), HumanMessage(content=user)]
|
||||
):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
writer({"type": "intent_chunk", "text": reasoning, "is_thinking": True})
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
@@ -259,6 +264,12 @@ async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> d
|
||||
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
|
||||
|
||||
updates: dict = {"status": "researching", "progress_events": progress_events}
|
||||
# 捕获用户表单选定的主模型进 state(仅本节点能可靠拿到 configurable.model_name —
|
||||
# 它运行在首次 invocation;后续大纲/写作/审核在 resume 后跑,config 已不带它)。
|
||||
# 大纲/写作/审核经 get_writing_model 严格读 state['model_name'],不回退默认。
|
||||
selected_model = (configurable.get("model_name") or "").strip()
|
||||
if selected_model:
|
||||
updates["model_name"] = selected_model
|
||||
updates["article_type"] = final_article_type
|
||||
updates["target_audience"] = final_target_audience
|
||||
updates["word_count_target"] = final_word_count
|
||||
|
||||
+125
-8
@@ -1,9 +1,11 @@
|
||||
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
|
||||
"""素材收集专家节点:关键词提取 → 并发搜索(含笔记) → 相关度排序。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
@@ -20,6 +22,10 @@ from ._utils import get_ai_writing_config, llm_json, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_NOTEBOOK_SEARCH_PATH = "/api/v1/source/mentioned-documents/search"
|
||||
_NOTEBOOK_MAX_KEYWORDS = 3 # 每次写作最多对笔记发起的检索轮数
|
||||
_NOTEBOOK_TOP_K = 5 # 每次检索取回的 chunk 数
|
||||
|
||||
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 50
|
||||
_DEFAULT_MAX_MATERIALS = 50
|
||||
_MIN_MAX_MATERIALS = 1
|
||||
@@ -29,6 +35,11 @@ _MAX_MAX_MATERIALS = 100
|
||||
_DEFAULT_KEYWORD_COUNT = 6
|
||||
_MAX_KEYWORD_COUNT = 10
|
||||
|
||||
# 单个技能的 web 检索超时(秒)。技能检索是顺序循环、最多 5 个技能,
|
||||
# DuckDuckGo 等 provider 偶发限速/挂起会把整段"技能检索过程"拖到几十秒看起来卡死;
|
||||
# 超时即放弃该技能、emit 0 条结果继续下一个,保证时间线持续推进。
|
||||
_SKILL_SEARCH_TIMEOUT_SECONDS = 25
|
||||
|
||||
|
||||
def _resolve_keyword_count(raw: object) -> int:
|
||||
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
|
||||
@@ -89,6 +100,67 @@ def _notice(title: str, detail: str, raw: str = "") -> dict:
|
||||
return event
|
||||
|
||||
|
||||
async def _search_notebook(
|
||||
keywords: list[str],
|
||||
search_space_id: int,
|
||||
source_ids: list[int],
|
||||
token: str,
|
||||
) -> list[Material]:
|
||||
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
|
||||
import httpx # 懒导入:仅在笔记模式下才需要
|
||||
|
||||
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
|
||||
if not base:
|
||||
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
|
||||
return []
|
||||
|
||||
results: list[Material] = []
|
||||
mat_idx = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{base}{_NOTEBOOK_SEARCH_PATH}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"query": kw,
|
||||
"source_ids": source_ids,
|
||||
"search_space_id": search_space_id,
|
||||
"top_k": _NOTEBOOK_TOP_K,
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
# 解析 documents 列表,每个 chunk 作为独立素材
|
||||
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
|
||||
for doc in documents or []:
|
||||
doc_title: str = doc.get("title") or "笔记文档"
|
||||
doc_url: str = doc.get("url") or ""
|
||||
for chunk in doc.get("chunks") or []:
|
||||
chunk_content: str = (chunk.get("content") or "").strip()
|
||||
if not chunk_content:
|
||||
continue
|
||||
mat_idx += 1
|
||||
raw_score = chunk.get("score")
|
||||
chunk_score = float(raw_score) if raw_score is not None else max(0.5, 0.95 - (mat_idx - 1) * 0.03)
|
||||
results.append(Material(
|
||||
id=f"nb{mat_idx:03d}",
|
||||
title=doc_title,
|
||||
content=chunk_content[:500],
|
||||
source=f"笔记·{doc_title}",
|
||||
source_type="notebook",
|
||||
relevance_score=chunk_score,
|
||||
url=doc_url,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning("notebook search kw=%r failed: %s", kw, e)
|
||||
except Exception as e:
|
||||
logger.warning("notebook search client error: %s", e)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
intent = state["user_intent"]
|
||||
article_type = state.get("article_type", "analysis")
|
||||
@@ -110,11 +182,19 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
is_user_query_search = action == "re_search" and bool(user_query)
|
||||
# 兼容旧版"追加关键词"调用方(如果还有 client 传 extra_keywords 数组而非 user_query)
|
||||
is_append_only = action == "re_search" and not user_query and bool(extra_keywords)
|
||||
# PAUSE-1 素材确认卡用户点「启用技能检索」:
|
||||
# 用户在写作表单选择的素材来源:knowledge_base(默认 web/intranet 检索)/ skill(技能检索)。
|
||||
# 首次检索(没有用户干预)时按它决定检索方式;缺省 knowledge_base。
|
||||
material_source = (state.get("material_source") or "knowledge_base").strip()
|
||||
is_first_run = not intervention
|
||||
is_initial_skill_source = is_first_run and material_source == "skill"
|
||||
# 技能检索触发条件(任一即可):
|
||||
# - 首次检索且表单选了「技能检索」来源
|
||||
# - PAUSE-1 素材确认卡用户点「启用技能检索」按钮
|
||||
# 技能检索流程:
|
||||
# - 不再走 web/intranet 检索,改用 SkillSearchProvider
|
||||
# - 顺序调用候选技能 + 早停(第一个技能够就停,不够再加下一个)
|
||||
# - 旧 material_package 完全丢弃(覆盖式替换,让用户对比 web vs skill 结果)
|
||||
is_skill_search = action == "enable_skill_search"
|
||||
is_skill_search = action == "enable_skill_search" or is_initial_skill_source
|
||||
|
||||
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
|
||||
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
|
||||
@@ -167,6 +247,22 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
|
||||
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
|
||||
notebook_materials: list[Material] = []
|
||||
if notebook_search_space_id:
|
||||
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
|
||||
notebook_token: str = cfg.get("notebook_token") or ""
|
||||
_emit("正在检索笔记内容…")
|
||||
notebook_materials = await _search_notebook(
|
||||
keywords[:_NOTEBOOK_MAX_KEYWORDS],
|
||||
notebook_search_space_id,
|
||||
notebook_source_ids,
|
||||
notebook_token,
|
||||
)
|
||||
if notebook_materials:
|
||||
_emit(f"从笔记中获取到 {len(notebook_materials)} 条素材")
|
||||
|
||||
# Step 2: 搜索
|
||||
# 技能检索:技能不是素材,是"检索策略 / 视角"。流程:
|
||||
# a. SkillPicker 选出按相关度排序的 skills(推 list_skills + select_skills 事件)
|
||||
@@ -178,6 +274,10 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
|
||||
def _emit_skill_step(step: dict) -> None:
|
||||
skill_steps.append(step)
|
||||
# 技能检索时机 trace:grep "[skill-timing]" 看每步 emit 的真实时刻,
|
||||
# 与前端 [ai-writing:custom] 日志对齐,钉死是否实时推送。
|
||||
logger.info("[skill-timing] t=%.3f emit skill_search_step type=%s skill=%s",
|
||||
time.monotonic(), step.get("type"), step.get("skill_name") or "")
|
||||
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
|
||||
|
||||
if is_skill_search:
|
||||
@@ -205,15 +305,25 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
|
||||
for entry in selected:
|
||||
sk = entry["skill"]
|
||||
# 用 LLM 把 user_query + skill.description 转成针对性关键词
|
||||
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
|
||||
# invoke_skill 步骤带上实际检索词(截断防过长),让用户在每次长 await 之前
|
||||
# 就看到"这一步在用技能 X 检索什么",配合前端 spinner 体现进行中。
|
||||
_emit_skill_step({
|
||||
"type": "invoke_skill",
|
||||
"message": f"使用技能「{sk.name}」开始检索…",
|
||||
"message": f"使用技能「{sk.name}」检索:{skill_query[:60]}",
|
||||
"skill_name": sk.name,
|
||||
})
|
||||
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
|
||||
try:
|
||||
res = await web_provider.search(skill_query, max_results=per_skill)
|
||||
# 加超时兜底:单个技能检索最多等 _SKILL_SEARCH_TIMEOUT_SECONDS 秒,
|
||||
# 超时不再阻塞后续技能,避免整段过程看起来"冻结"。
|
||||
res = await asyncio.wait_for(
|
||||
web_provider.search(skill_query, max_results=per_skill),
|
||||
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("skill[%s] search timed out after %ss", sk.name, _SKILL_SEARCH_TIMEOUT_SECONDS)
|
||||
search_errors.append(f"skill[{sk.name}]: 检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)")
|
||||
res = []
|
||||
except Exception as e:
|
||||
logger.warning("skill[%s] search failed: %s", sk.name, e)
|
||||
search_errors.append(f"skill[{sk.name}]: {e}")
|
||||
@@ -302,10 +412,17 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
existing_pkg = state.get("material_package")
|
||||
if is_skill_search:
|
||||
# 技能编排循环已经完成素材累积,直接用 raw_materials_pre 当 raw_materials
|
||||
# 笔记素材追加在技能素材之后
|
||||
raw_materials: list[Material] = raw_materials_pre # type: ignore[name-defined]
|
||||
raw_materials.extend(notebook_materials)
|
||||
else:
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials = []
|
||||
# 笔记素材优先排在最前面,赋予高相关度
|
||||
raw_materials = list(notebook_materials)
|
||||
for m in notebook_materials:
|
||||
if m.get("url"):
|
||||
seen_urls.add(m["url"])
|
||||
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only)
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
|
||||
+13
-2
@@ -12,7 +12,7 @@ from ..prompts.writer_prompts import (
|
||||
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
|
||||
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
|
||||
)
|
||||
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser, safe_get_stream_writer
|
||||
from ._utils import get_writing_model, _content_to_text, count_words, reasoning_delta, ThinkTagParser, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -197,7 +197,7 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
model = get_writing_model(config, state)
|
||||
except Exception as e:
|
||||
logger.error("draft model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
@@ -250,6 +250,17 @@ async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> di
|
||||
SystemMessage(content=DRAFT_SECTION_SYSTEM),
|
||||
HumanMessage(content=user_msg),
|
||||
]):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
writer({
|
||||
"type": "draft_chunk",
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": reasoning,
|
||||
"is_thinking": True,
|
||||
"is_section_done": False,
|
||||
})
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
|
||||
+126
-5
@@ -7,11 +7,19 @@ import logging
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, KeyPoint, Outline, OutlineSection
|
||||
from ..outline_parser import parse_user_outline
|
||||
from ..prompts.writer_prompts import (
|
||||
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
|
||||
OUTLINE_MATERIAL_ASSIGN_SYSTEM,
|
||||
OUTLINE_MATERIAL_ASSIGN_USER,
|
||||
OUTLINE_NORMALIZE_SYSTEM,
|
||||
OUTLINE_NORMALIZE_USER,
|
||||
OUTLINE_REVISE_SYSTEM,
|
||||
OUTLINE_REVISE_USER,
|
||||
OUTLINE_SYSTEM,
|
||||
OUTLINE_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser, safe_get_stream_writer
|
||||
from ..state import AIWritingState, KeyPoint, Outline, OutlineSection
|
||||
from ._utils import ThinkTagParser, _content_to_text, get_writing_model, llm_json, parse_json, reasoning_delta, safe_get_stream_writer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -80,6 +88,108 @@ def _parse_outline(raw: dict) -> Outline:
|
||||
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
|
||||
|
||||
|
||||
def _derive_title(state: AIWritingState) -> str:
|
||||
"""用户大纲没给总标题时,用写作意图兜底一个标题。"""
|
||||
intent = (state.get("user_intent") or "").strip()
|
||||
if not intent:
|
||||
return "未命名文章"
|
||||
return intent[:30].rstrip() + ("…" if len(intent) > 30 else "")
|
||||
|
||||
|
||||
async def _normalize_user_outline_via_llm(state: AIWritingState, config: RunnableConfig, raw: str) -> Outline:
|
||||
"""确定性解析失败时,调 LLM 把用户大纲原文归一成三级结构(严格保真、不改写)。
|
||||
|
||||
再失败则把每个非空行当作一个章节,保证永远产出一份可用大纲、不丢用户内容。
|
||||
"""
|
||||
model_name = (state.get("model_name") or "").strip() or None
|
||||
line_fallback = {
|
||||
"title": "",
|
||||
"sections": [
|
||||
{"section_title": ln.strip(), "key_points": [], "material_ids": []}
|
||||
for ln in raw.splitlines() if ln.strip()
|
||||
] or [{"section_title": "正文", "key_points": [], "material_ids": []}],
|
||||
}
|
||||
result = await llm_json(
|
||||
OUTLINE_NORMALIZE_SYSTEM,
|
||||
OUTLINE_NORMALIZE_USER.format(raw_outline=raw),
|
||||
line_fallback,
|
||||
config,
|
||||
model_name=model_name,
|
||||
)
|
||||
return _parse_outline(result)
|
||||
|
||||
|
||||
async def _assign_materials_to_outline(
|
||||
state: AIWritingState, config: RunnableConfig, outline: Outline, pkg: dict
|
||||
) -> Outline:
|
||||
"""为用户大纲的各章节回填 material_ids(结构锁定:只取编号、按章节序号嫁接)。
|
||||
|
||||
LLM 只负责「哪个章节配哪些素材」,标题与层级一律以解析结果为准,杜绝改写。失败则
|
||||
保持 material_ids 为空(writer_draft 自带「取前 2 条素材」兜底)。
|
||||
"""
|
||||
sections = outline.get("sections", [])
|
||||
materials = pkg.get("materials") or []
|
||||
if not sections or not materials:
|
||||
return outline
|
||||
|
||||
valid_ids = {m["id"] for m in materials}
|
||||
sections_list = "\n".join(
|
||||
f"{i + 1}. {s.get('section_title', '')}" for i, s in enumerate(sections)
|
||||
)
|
||||
model_name = (state.get("model_name") or "").strip() or None
|
||||
result = await llm_json(
|
||||
OUTLINE_MATERIAL_ASSIGN_SYSTEM,
|
||||
OUTLINE_MATERIAL_ASSIGN_USER.format(
|
||||
user_intent=state.get("user_intent", ""),
|
||||
sections_list=sections_list,
|
||||
materials_summary=_materials_summary(pkg),
|
||||
),
|
||||
{"assignments": []},
|
||||
config,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
assignments = result.get("assignments")
|
||||
if not isinstance(assignments, list):
|
||||
return outline
|
||||
for i, section in enumerate(sections):
|
||||
if i >= len(assignments):
|
||||
break
|
||||
ids = assignments[i]
|
||||
if isinstance(ids, list):
|
||||
section["material_ids"] = [mid for mid in ids if mid in valid_ids]
|
||||
return outline
|
||||
|
||||
|
||||
async def _build_outline_from_user(
|
||||
state: AIWritingState, config: RunnableConfig, pkg: dict, raw: str
|
||||
) -> dict:
|
||||
"""用户自带大纲分支:解析用户大纲 → 关联素材 → 直接进确认,不调 LLM 重新规划。"""
|
||||
writer = safe_get_stream_writer()
|
||||
writer({"type": "outline_chunk", "text": "正在整理你提供的大纲…\n", "is_thinking": True})
|
||||
|
||||
outline = parse_user_outline(raw)
|
||||
if outline is None:
|
||||
# 确定性解析拿不准(散文化/格式混乱)→ LLM 兜底归一
|
||||
outline = await _normalize_user_outline_via_llm(state, config, raw)
|
||||
|
||||
if not outline.get("title"):
|
||||
outline = {**outline, "title": _derive_title(state)}
|
||||
|
||||
# 仍走了检索 → 把已检索素材关联到用户章节(结构锁定)
|
||||
outline = await _assign_materials_to_outline(state, config, outline, pkg)
|
||||
|
||||
section_count = len(outline.get("sections", []))
|
||||
return {
|
||||
"current_outline": outline,
|
||||
"status": "awaiting_outline_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": "采用你提供的大纲"},
|
||||
{"type": "outline_ready", "agent_name": "作家", "message": f"已整理你的大纲,共 {section_count} 个章节"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
@@ -95,6 +205,13 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
current_outline = state.get("current_outline")
|
||||
is_revise = bool(feedback) and bool(current_outline)
|
||||
|
||||
# 用户自带大纲:首轮(尚无大纲、非重规划)且表单粘了大纲原文 → 走「采用用户大纲」分支,
|
||||
# 跳过 LLM 重新规划。后续若用户在确认卡里点「重新规划」,current_outline 已存在 → 落到
|
||||
# 下方 is_revise 编辑模式,按反馈调整用户大纲(符合预期)。
|
||||
user_outline_raw = (state.get("user_outline_raw") or "").strip()
|
||||
if user_outline_raw and not current_outline and not is_revise:
|
||||
return await _build_outline_from_user(state, config, pkg, user_outline_raw)
|
||||
|
||||
if is_revise:
|
||||
system_prompt = OUTLINE_REVISE_SYSTEM
|
||||
user_prompt = OUTLINE_REVISE_USER.format(
|
||||
@@ -125,13 +242,17 @@ async def writer_outline_node(state: AIWritingState, config: RunnableConfig) ->
|
||||
parser = ThinkTagParser()
|
||||
|
||||
try:
|
||||
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# get_writing_model 放进 try:用户模型缺失 / 模型名配错时抛 ValueError,
|
||||
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
|
||||
model = get_model(config)
|
||||
model = get_writing_model(config, state)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=system_prompt),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
writer({"type": "outline_chunk", "text": reasoning, "is_thinking": True})
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
|
||||
+220
@@ -0,0 +1,220 @@
|
||||
"""把用户粘贴的大纲文本解析成三级 ``Outline`` 结构(确定性解析,不依赖 LLM)。
|
||||
|
||||
用户在写作表单里可能直接粘贴一份已有大纲,要求「完全按我的大纲生成」。本模块负责
|
||||
把各种常见的大纲文本格式严格、保真地转成后端三级结构:
|
||||
|
||||
Outline.sections[] = Level 1 章节(section_title)
|
||||
OutlineSection.key_points[] = Level 2 小节(point_title)
|
||||
KeyPoint.sub_points[] = Level 3 子要点
|
||||
|
||||
兼容的输入格式:
|
||||
- Markdown 标题:``#`` / ``##`` / ``###`` / ``####``(首个孤立 ``#`` 视为文章标题)
|
||||
- 中文编号:``一、`` / ``(一)`` / ``第一章`` / ``第一节``
|
||||
- 阿拉伯编号:``1.`` / ``1、`` / ``1.1`` / ``1.1.1``
|
||||
- 缩进 bullet:``-`` / ``*`` / ``•`` / ``·``(用缩进深度表示层级)
|
||||
- 纯文本平铺:每行一个一级标题(短标题、无句末标点)
|
||||
|
||||
设计原则:
|
||||
- **保真不改写**:只识别层级、原样保留标题文本,绝不增删或改写用户内容。
|
||||
- **多级混排自适应**:把文本里实际出现的各种「层级标记」收集起来排序,按相对外层
|
||||
顺序映射到 1/2/3 级——所以不依赖固定的标记搭配,``##``+``-`` 或 ``一、``+缩进
|
||||
都能正确分层。
|
||||
- **拿不准就交给上层**:识别不出可靠层级(如整段散文)时返回 ``None``,由调用方走
|
||||
LLM 兜底归一,避免硬把散文拆成错误结构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from .state import KeyPoint, Outline, OutlineSection
|
||||
|
||||
# ── 行首标记识别 ──────────────────────────────────────────────────────────────
|
||||
|
||||
_CN_NUM = "一二三四五六七八九十百千零〇两"
|
||||
|
||||
# Markdown 标题:## 标题
|
||||
_MD_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*\S)\s*$")
|
||||
# 第一章 / 第1节 / 第一部分
|
||||
_CN_CHAPTER_RE = re.compile(rf"^第\s*[{_CN_NUM}\d]+\s*[章节部分篇讲回卷](?:[\s::、..))]\s*|\s+)?(.*)$")
|
||||
# 一、 / 十二、 / 一. / 一.
|
||||
_CN_DUN_RE = re.compile(rf"^[{_CN_NUM}]+\s*[、..]\s*(.*)$")
|
||||
# (一) / (一) / (1) / (1)
|
||||
_PAREN_RE = re.compile(rf"^[((]\s*[{_CN_NUM}\d]+\s*[))]\s*(.*)$")
|
||||
# 1.1 / 1.1.1(带至少一个点的多级阿拉伯编号)
|
||||
_ARABIC_DOTTED_RE = re.compile(r"^(\d+(?:\.\d+)+)\s*[、..))]?\s*(.*)$")
|
||||
# 1. / 1、 / 1) / 1.(单级阿拉伯编号)
|
||||
_ARABIC_SINGLE_RE = re.compile(r"^\d+\s*[、..))]\s*(.*)$")
|
||||
# 带圈数字 ①②③ / 罗马 ⅰ
|
||||
_CIRCLED_RE = re.compile(r"^[①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮]\s*(.*)$")
|
||||
# bullet:- / * / • / ·(要求标记后有空白,避免误吞「*强调*」之类)
|
||||
_BULLET_RE = re.compile(r"^[-*•·]\s+(.*\S)\s*$")
|
||||
|
||||
# 标记分组的「外层度」基准值(越小越外层)。group: 0=标题 1=编号 2=缩进。
|
||||
# 同组内 order 决定相对层级;不同组之间靠 group 排序(标题 < 编号 < 缩进)。
|
||||
# 编号顺序遵循中文公文惯例:第一章 > 一、 > (一) > 1. > 1.1 > ①。
|
||||
_NUM_ORDER = {
|
||||
"cn_chapter": 0.0, # 第一章
|
||||
"cn_dun": 1.0, # 一、
|
||||
"paren": 2.0, # (一)
|
||||
"arabic_single": 3.0, # 1.
|
||||
"circled": 8.0, # ①
|
||||
}
|
||||
|
||||
|
||||
def _indent_width(line: str) -> int:
|
||||
"""计算行首缩进宽度(tab 记 4 列)。"""
|
||||
width = 0
|
||||
for ch in line:
|
||||
if ch == " ":
|
||||
width += 1
|
||||
elif ch == "\t":
|
||||
width += 4
|
||||
else:
|
||||
break
|
||||
return width
|
||||
|
||||
|
||||
def _classify(stripped: str, indent: int) -> tuple[tuple[int, float], str] | None:
|
||||
"""识别一行的层级标记,返回 ``((group, order), 纯文本)``;无法识别返回 None。
|
||||
|
||||
``(group, order)`` 是该行的「层级键」:把全文出现过的不同键排序后,按位置映射到
|
||||
1/2/3 级。这样不必预设标记搭配,相对顺序对了层级就对。
|
||||
"""
|
||||
if not stripped:
|
||||
return None
|
||||
|
||||
# 1) Markdown 标题(group 0,order = 井号数)
|
||||
m = _MD_HEADING_RE.match(stripped)
|
||||
if m:
|
||||
depth = len(m.group(1))
|
||||
text = m.group(2).strip()
|
||||
return (0, float(depth)), text
|
||||
|
||||
# 2) 各类编号(group 1)
|
||||
m = _CN_CHAPTER_RE.match(stripped)
|
||||
if m and stripped.startswith("第"):
|
||||
return (1, _NUM_ORDER["cn_chapter"]), (m.group(1) or "").strip()
|
||||
m = _CN_DUN_RE.match(stripped)
|
||||
if m:
|
||||
return (1, _NUM_ORDER["cn_dun"]), m.group(1).strip()
|
||||
m = _ARABIC_DOTTED_RE.match(stripped)
|
||||
if m:
|
||||
depth = m.group(1).count(".") + 1 # 1.1 → 2, 1.1.1 → 3
|
||||
# 多级阿拉伯:order 随层级递增,落在单级(3.0)与圈号(8.0)之间
|
||||
return (1, 3.0 + depth), m.group(2).strip()
|
||||
m = _PAREN_RE.match(stripped)
|
||||
if m:
|
||||
return (1, _NUM_ORDER["paren"]), m.group(1).strip()
|
||||
m = _ARABIC_SINGLE_RE.match(stripped)
|
||||
if m:
|
||||
return (1, _NUM_ORDER["arabic_single"]), m.group(1).strip()
|
||||
m = _CIRCLED_RE.match(stripped)
|
||||
if m:
|
||||
return (1, _NUM_ORDER["circled"]), m.group(1).strip()
|
||||
|
||||
# 3) bullet:用缩进深度区分层级(group 2)
|
||||
m = _BULLET_RE.match(stripped)
|
||||
if m:
|
||||
return (2, float(indent)), m.group(1).strip()
|
||||
|
||||
# 4) 纯文本:用缩进深度区分层级(group 3,最内层基准)
|
||||
return (3, float(indent)), stripped
|
||||
|
||||
|
||||
def _looks_like_prose(text: str) -> bool:
|
||||
"""判断一行是否更像散文句子而非大纲标题(长、或带句末标点)。"""
|
||||
if len(text) > 50:
|
||||
return True
|
||||
return text.rstrip()[-1:] in "。!?!?;;"
|
||||
|
||||
|
||||
def parse_user_outline(raw: str) -> Outline | None:
|
||||
"""把用户粘贴的大纲文本解析成三级 ``Outline``;无法可靠解析时返回 ``None``。
|
||||
|
||||
返回 ``None`` 的情形:空白输入、只有一行且无任何标记、或整体更像散文段落——这些交给
|
||||
上层用 LLM 兜底归一,避免误拆。
|
||||
"""
|
||||
if not raw or not raw.strip():
|
||||
return None
|
||||
|
||||
rows: list[tuple[tuple[int, float], str]] = []
|
||||
for line in raw.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
indent = _indent_width(line)
|
||||
classified = _classify(line.strip(), indent)
|
||||
if classified is None:
|
||||
continue
|
||||
key, text = classified
|
||||
if text:
|
||||
rows.append((key, text))
|
||||
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
marked = any(key[0] != 3 for key, _ in rows) # 是否出现过标题/编号/bullet 标记
|
||||
|
||||
# 纯文本且无任何标记:可能是「平铺标题列表」也可能是散文。
|
||||
# 单行、或任一行像句子 → 判为散文,返回 None 交给 LLM。
|
||||
if not marked:
|
||||
if len(rows) < 2:
|
||||
return None
|
||||
if any(_looks_like_prose(text) for _, text in rows):
|
||||
return None
|
||||
|
||||
# 文章标题:仅在「Markdown 首行是孤立的最浅井号、且存在更深井号」时认定。
|
||||
title = ""
|
||||
body = rows
|
||||
heading_depths = [key[1] for key, _ in rows if key[0] == 0]
|
||||
if heading_depths:
|
||||
min_depth = min(heading_depths)
|
||||
first_key = rows[0][0]
|
||||
is_first_top_heading = first_key == (0, min_depth)
|
||||
only_one_at_top = sum(1 for d in heading_depths if d == min_depth) == 1
|
||||
has_deeper = any(d > min_depth for d in heading_depths)
|
||||
if is_first_top_heading and only_one_at_top and has_deeper:
|
||||
title = rows[0][1]
|
||||
body = rows[1:]
|
||||
|
||||
if not body:
|
||||
return None
|
||||
|
||||
# 把 body 里出现的不同「层级键」排序,按位置映射到 1/2/3 级(超过 3 级压到 3)。
|
||||
distinct_keys = sorted({key for key, _ in body})
|
||||
level_of: dict[tuple[int, float], int] = {
|
||||
key: min(idx + 1, 3) for idx, key in enumerate(distinct_keys)
|
||||
}
|
||||
|
||||
sections: list[OutlineSection] = []
|
||||
cur_section: OutlineSection | None = None
|
||||
cur_point: KeyPoint | None = None
|
||||
|
||||
for key, text in body:
|
||||
level = level_of[key]
|
||||
if level <= 1:
|
||||
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
|
||||
sections.append(cur_section)
|
||||
cur_point = None
|
||||
elif level == 2:
|
||||
if cur_section is None: # 孤立 L2:提升为章节,避免空标题
|
||||
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
|
||||
sections.append(cur_section)
|
||||
cur_point = None
|
||||
continue
|
||||
cur_point = KeyPoint(point_title=text, sub_points=[])
|
||||
cur_section["key_points"].append(cur_point)
|
||||
else: # level == 3
|
||||
if cur_section is None: # 孤立 L3:提升为章节
|
||||
cur_section = OutlineSection(section_title=text, key_points=[], material_ids=[])
|
||||
sections.append(cur_section)
|
||||
cur_point = None
|
||||
elif cur_point is None: # 章节下尚无小节:提升为小节
|
||||
cur_point = KeyPoint(point_title=text, sub_points=[])
|
||||
cur_section["key_points"].append(cur_point)
|
||||
else:
|
||||
cur_point["sub_points"].append(text)
|
||||
|
||||
if not sections:
|
||||
return None
|
||||
|
||||
return Outline(title=title, sections=sections)
|
||||
+54
@@ -123,6 +123,60 @@ OUTLINE_REVISE_USER = """写作意图:{user_intent}
|
||||
|
||||
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
|
||||
|
||||
# ── 用户自带大纲:LLM 兜底归一(确定性解析失败时才用)──────────────────────────
|
||||
|
||||
OUTLINE_NORMALIZE_SYSTEM = """你是一名大纲格式整理助手。
|
||||
用户粘贴了一份已有的文章大纲(格式可能不规范),你的任务是把它原样转成结构化 JSON。
|
||||
|
||||
【最高原则:保真,绝不改写】
|
||||
- 严格保留用户写的每一条标题文字,逐字照搬,不要润色、改写、翻译、增加或删除任何条目。
|
||||
- 你只判断层级(哪条是章节、哪条是小节、哪条是子要点),不创作内容。
|
||||
- 层级判断依据用户原文里的编号(一、/(一)/1./1.1)、Markdown 井号数、或缩进深度。
|
||||
|
||||
【三级结构】
|
||||
- Level 1 章节 → section_title
|
||||
- Level 2 小节 → key_points[].point_title
|
||||
- Level 3 子要点 → key_points[].sub_points[](字符串数组)
|
||||
- 用户大纲只有一级时,每条都是章节,key_points 留空 []。
|
||||
- 用户大纲只有两级时,sub_points 留空 []。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要解释文字,不要 markdown 代码块,必须用英文双引号。
|
||||
- material_ids 一律输出 [](此步不关联素材)。
|
||||
- 结构:{"title": "...", "sections": [{"section_title": "...", "key_points": [{"point_title": "...", "sub_points": ["..."]}], "material_ids": []}]}
|
||||
- 用户大纲若包含一个总标题,放进 title;没有就留空字符串。"""
|
||||
|
||||
OUTLINE_NORMALIZE_USER = """用户粘贴的大纲原文:
|
||||
{raw_outline}
|
||||
|
||||
请逐条保留原文,仅判断层级,输出结构化大纲 JSON。"""
|
||||
|
||||
# ── 用户自带大纲:素材关联(结构锁定,只挑 material_ids)──────────────────────────
|
||||
|
||||
OUTLINE_MATERIAL_ASSIGN_SYSTEM = """你是一名资料编辑。
|
||||
给定一份【已确定的文章大纲】(章节顺序固定)和一批检索到的素材,你的任务是为每个章节
|
||||
挑选最相关的素材编号。
|
||||
|
||||
【规则】
|
||||
- 不要修改大纲的任何标题或结构,你只输出每个章节对应的素材编号列表。
|
||||
- 严格按大纲给出的章节顺序输出,一个章节对应一个编号数组;该章节没有相关素材就给 []。
|
||||
- material_id 必须来自给定素材清单里出现过的编号,不要编造。
|
||||
- 一条素材可以分配给多个章节,也可以一个都不分配。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要解释文字,不要 markdown 代码块,必须用英文双引号。
|
||||
- 结构:{"assignments": [["m001","m002"], [], ["m003"]]},数组长度等于章节数、顺序一致。"""
|
||||
|
||||
OUTLINE_MATERIAL_ASSIGN_USER = """写作意图:{user_intent}
|
||||
|
||||
文章大纲(章节按顺序编号):
|
||||
{sections_list}
|
||||
|
||||
可用素材:
|
||||
{materials_summary}
|
||||
|
||||
请按章节顺序,为每个章节挑选相关的 material_id,输出 assignments 数组。"""
|
||||
|
||||
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
|
||||
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
|
||||
|
||||
+24
@@ -16,6 +16,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
@@ -26,6 +27,7 @@ from ..nodes._utils import (
|
||||
_content_to_text,
|
||||
get_model,
|
||||
parse_json,
|
||||
reasoning_delta,
|
||||
safe_get_stream_writer,
|
||||
)
|
||||
|
||||
@@ -127,12 +129,26 @@ class SkillPicker:
|
||||
|
||||
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
|
||||
|
||||
# list_skills 之后立刻发一个"分析中"提示步骤,避免 LLM 第一个 chunk 到来前
|
||||
# 用户看到空窗期(某些模型 astream 在头部 buffer 时间较长)
|
||||
self._emit_step({
|
||||
"type": "match_skills",
|
||||
"message": "正在让模型从候选技能中筛选最合适的…",
|
||||
})
|
||||
|
||||
# 流式 LLM 调用:与 intent_parser / writer_outline 同套机制
|
||||
writer = safe_get_stream_writer()
|
||||
parser = ThinkTagParser()
|
||||
full_text = ""
|
||||
|
||||
_chunk_count = {"n": 0}
|
||||
|
||||
def _emit_chunk(text: str, is_thinking: bool) -> None:
|
||||
# 只在首个 chunk 打一条 timing,避免逐 token 刷屏;用来和 skill_search_step
|
||||
# 的 [skill-timing] 对比"选择打字"与"步骤事件"谁先到。
|
||||
if _chunk_count["n"] == 0:
|
||||
logger.info("[skill-timing] t=%.3f first skill_select_chunk", time.monotonic())
|
||||
_chunk_count["n"] += 1
|
||||
writer({
|
||||
"type": "skill_select_chunk",
|
||||
"text": text,
|
||||
@@ -140,6 +156,10 @@ class SkillPicker:
|
||||
"agent_name": "素材收集专家",
|
||||
})
|
||||
|
||||
# 立刻推一个空 chunk,让前端 streaming view 立即挂载(占位 + 闪烁光标),
|
||||
# 即使后续 LLM 首个 chunk 到达较慢,UI 也不再静默
|
||||
_emit_chunk("", False)
|
||||
|
||||
try:
|
||||
model = get_model(self._config, model_name=self._model_name)
|
||||
async for chunk in model.astream([
|
||||
@@ -152,6 +172,10 @@ class SkillPicker:
|
||||
max_select=self._max_select,
|
||||
)),
|
||||
]):
|
||||
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
|
||||
reasoning = reasoning_delta(chunk)
|
||||
if reasoning:
|
||||
_emit_chunk(reasoning, True)
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
|
||||
@@ -125,6 +125,9 @@ class BlockedSection(TypedDict):
|
||||
class AIWritingState(TypedDict):
|
||||
# 写作配置
|
||||
user_intent: str
|
||||
# 用户在写作表单粘贴的「已有大纲」原文(可选)。非空时 writer_outline 跳过 LLM 规划,
|
||||
# 直接解析这份大纲并按用户结构生成,做到「完全按我的大纲写」。留空走正常自动规划。
|
||||
user_outline_raw: str
|
||||
article_type: str
|
||||
target_audience: str
|
||||
word_count_target: int
|
||||
@@ -132,6 +135,17 @@ class AIWritingState(TypedDict):
|
||||
keyword_count: int
|
||||
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
|
||||
writing_mode: str
|
||||
# 素材来源:knowledge_base=知识库(默认 web/intranet 检索)/ skill=技能检索。
|
||||
# 用户在写作表单选择,首次检索时由 researcher 节点据此决定检索方式(缺省 knowledge_base)。
|
||||
material_source: str
|
||||
# 用户在写作表单选定的主模型。大纲/写作/审核严格使用它(不回退默认)。
|
||||
# 由 intent_parser 在首次 invocation 从 configurable.model_name 捕获进 state,
|
||||
# 经 checkpoint 持久化 —— interrupt/resume 是新 invocation、其 config 不带原
|
||||
# model_name,存进 state 才能让 resume 后才运行的节点拿到用户模型。
|
||||
model_name: str
|
||||
# 笔记来源(可选):传入后 researcher 阶段会额外从笔记检索素材
|
||||
notebook_search_space_id: Optional[int]
|
||||
notebook_source_ids: Optional[List[int]]
|
||||
|
||||
# Agent A 输出
|
||||
material_package: Optional[MaterialPackage]
|
||||
|
||||
+48
-9
@@ -11,6 +11,7 @@ from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
|
||||
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
|
||||
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
|
||||
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
|
||||
from deerflow.agents.middlewares.setup_writing_approval_middleware import SetupWritingApprovalMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
@@ -25,6 +26,9 @@ from deerflow.models import create_chat_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 写作配置 Q&A:意图澄清最多几轮(一条用户消息 = 一轮),到顶强制提交配置卡。
|
||||
WRITING_SETUP_MAX_ROUNDS = 3
|
||||
|
||||
|
||||
def _get_runtime_config(config: RunnableConfig) -> dict:
|
||||
"""Merge legacy configurable options with LangGraph runtime context."""
|
||||
@@ -272,6 +276,7 @@ def _build_middlewares(
|
||||
# Add TodoList middleware if plan mode is enabled
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
is_writing_setup = cfg.get("is_writing_setup", False)
|
||||
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
|
||||
if todo_list_middleware is not None:
|
||||
middlewares.append(todo_list_middleware)
|
||||
@@ -280,18 +285,28 @@ def _build_middlewares(
|
||||
if resolved_app_config.token_usage.enabled:
|
||||
middlewares.append(TokenUsageMiddleware())
|
||||
|
||||
# Add TitleMiddleware
|
||||
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
|
||||
# Add TitleMiddleware — but skip it for scheduled runs. Each scheduled run
|
||||
# executes on a throwaway thread whose title is never shown to a user, and
|
||||
# generating one would fire an extra LLM call on the *default* model
|
||||
# (config.title.model_name is usually unset → models[0]), which is not the
|
||||
# model the task selected. Skipping it keeps scheduled/HTML runs on exactly
|
||||
# the chosen model.
|
||||
if not cfg.get("is_scheduled_run", False):
|
||||
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
|
||||
|
||||
# Add MemoryMiddleware (after TitleMiddleware) — skip in bootstrap mode (agent creation flow)
|
||||
memory_injection_enabled = cfg.get("memory_injection_enabled", True)
|
||||
if not is_bootstrap:
|
||||
if not is_bootstrap and not is_writing_setup:
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
|
||||
|
||||
# Add ViewImageMiddleware only if the current model supports vision.
|
||||
# Use the resolved runtime model_name from make_lead_agent to avoid stale config values.
|
||||
model_config = resolved_app_config.get_model_config(model_name) if model_name else None
|
||||
if model_config is not None and model_config.supports_vision:
|
||||
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
|
||||
# ingests raw image data). In delegated mode the view_image tool already
|
||||
# returns a text description and stores nothing in `viewed_images`, so the
|
||||
# middleware would otherwise inject a spurious "No images have been viewed."
|
||||
# message. Use the resolved runtime model_name to avoid stale config values.
|
||||
from deerflow.models.vision import is_delegated_vision, vision_enabled
|
||||
|
||||
if vision_enabled(model_name, app_config=resolved_app_config) and not is_delegated_vision(app_config=resolved_app_config):
|
||||
middlewares.append(ViewImageMiddleware())
|
||||
|
||||
# Add DeferredToolFilterMiddleware to hide deferred tool schemas from model binding
|
||||
@@ -314,6 +329,7 @@ def _build_middlewares(
|
||||
middlewares.extend(custom_middlewares)
|
||||
|
||||
middlewares.append(SetupAgentApprovalMiddleware())
|
||||
middlewares.append(SetupWritingApprovalMiddleware())
|
||||
|
||||
# SkillStopMiddleware — intercepts skill invocations and halts execution
|
||||
# when a configured skill is about to run. Request-level options
|
||||
@@ -361,7 +377,7 @@ def make_lead_agent(config: RunnableConfig):
|
||||
def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
# Lazy import to avoid circular dependency
|
||||
from deerflow.tools import get_available_tools
|
||||
from deerflow.tools.builtins import setup_agent
|
||||
from deerflow.tools.builtins import ask_clarification_tool, setup_agent, setup_writing
|
||||
|
||||
cfg = _get_runtime_config(config)
|
||||
resolved_app_config = app_config
|
||||
@@ -372,6 +388,10 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
# 写作配置 Q&A:与 is_bootstrap 同构的另一种「最小化 bootstrap」——挂 setup_writing 工具、
|
||||
# 走写作配置专用 prompt,只负责帮用户把写作表单聊清并提交,不真正开始写作。
|
||||
is_writing_setup = cfg.get("is_writing_setup", False)
|
||||
writing_setup_article_types = cfg.get("valid_article_types") if is_writing_setup else None
|
||||
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
|
||||
rag_mode_enabled = bool(cfg.get("rag_mode_enabled", False))
|
||||
writing_mode = bool(cfg.get("writing_mode", False))
|
||||
@@ -382,7 +402,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
requested_excluded_tools = cfg.get("excluded_tools") or None
|
||||
|
||||
try:
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap and not is_writing_setup else None
|
||||
except FileNotFoundError:
|
||||
logger.warning("Agent directory not found for '%s'; running with default config", agent_id)
|
||||
agent_config = None
|
||||
@@ -479,6 +499,25 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
}
|
||||
)
|
||||
|
||||
if is_writing_setup:
|
||||
# 写作配置 Q&A:最小化 agent —— 只挂 setup_writing(提交配置)+ ask_clarification(弹澄清卡)。
|
||||
# 关键:绝不给全套工具、绝不复用 lead 超级体提示,否则模型会把「新能源汽车市场分析」之类
|
||||
# 长得像任务的输入直接做掉,而不是转成写作配置提交。
|
||||
# 末尾追加 WritingSetupRoundCapMiddleware:澄清最多 3 轮,到顶用 tool_choice 强制提交配置卡。
|
||||
from deerflow.agents.lead_agent.prompt import build_writing_setup_prompt
|
||||
from deerflow.agents.middlewares.writing_setup_round_cap_middleware import WritingSetupRoundCapMiddleware
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=resolved_app_config),
|
||||
tools=[setup_writing, ask_clarification_tool],
|
||||
middleware=[
|
||||
*_build_middlewares(config, model_name=model_name, app_config=resolved_app_config),
|
||||
WritingSetupRoundCapMiddleware(max_rounds=WRITING_SETUP_MAX_ROUNDS),
|
||||
],
|
||||
system_prompt=build_writing_setup_prompt(writing_setup_article_types),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
|
||||
if is_bootstrap:
|
||||
# Special bootstrap agent with minimal prompt for initial custom agent creation flow
|
||||
return create_agent(
|
||||
|
||||
+58
-18
@@ -703,14 +703,14 @@ def _get_cached_skills_prompt_section(
|
||||
|
||||
display_pattern = (
|
||||
"""
|
||||
**Skill Result Display Pattern:**
|
||||
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
|
||||
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
|
||||
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
|
||||
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
|
||||
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
|
||||
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
|
||||
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
|
||||
**Skill Result Display Pattern:**
|
||||
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
|
||||
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
|
||||
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
|
||||
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
|
||||
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
|
||||
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
|
||||
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
|
||||
"""
|
||||
if display_signature
|
||||
else ""
|
||||
@@ -904,6 +904,46 @@ def _build_bootstrap_section() -> str:
|
||||
"""
|
||||
|
||||
|
||||
def build_writing_setup_prompt(article_types: list[dict] | None = None) -> str:
|
||||
"""写作配置 Q&A 的**独立**系统提示(不复用 lead-agent 超级体提示)。
|
||||
|
||||
关键:写作配置助手绝不能像 lead agent 那样「自己把任务做掉」——用户输入常常长得像一个
|
||||
可执行任务(如「新能源汽车市场分析」),若复用 lead 提示 + 全套工具,模型会直接去研究/写
|
||||
分析。这里用一份职责单一、口吻强硬的独立提示,配合「只挂 setup_writing 一个工具」,把模型
|
||||
死死框在「聊清需求 → 调 setup_writing」上。
|
||||
"""
|
||||
if article_types:
|
||||
type_lines = "\n".join(
|
||||
f" - `{t.get('key', '')}`:{t.get('label', '')}" for t in article_types if t.get("key")
|
||||
)
|
||||
type_block = f"【可选文章类型】article_type 只能填下列 key 之一:\n{type_lines}"
|
||||
else:
|
||||
type_block = "【文章类型】若不确定 article_type,可留空,交给用户在表单里选。"
|
||||
|
||||
return f"""你是「AI 写作」功能的**配置助手**。你的唯一职责:通过简短对话把用户的写作需求聊清楚,然后调用 `setup_writing` 工具把写作配置提交给用户确认。提交后由用户在表单里应用、再点「开始写作」。
|
||||
|
||||
【铁律——最高优先级】
|
||||
- 你**绝对不撰写**文章正文、分析内容、大纲内容或任何成稿文字。即使用户的输入看起来就是一个写作任务(例如「新能源汽车市场分析」「帮我写一篇……」),你也**只把它当作"要配置的写作目标"**,绝不替用户动笔、绝不直接完成这篇写作。
|
||||
- 你只有两个工具:① `ask_clarification`(向用户提问澄清,需要时用);② `setup_writing`(提交写作配置)。**没有任何其它工具**,不要做检索、不要做研究、不要假装调用别的工具。
|
||||
- 不要在回复里输出文章段落、要点清单或大纲(用户主动粘贴的大纲除外——原样放进 `outline` 参数)。
|
||||
|
||||
【工作流程】(与用户交互确认,但最多 3 轮内必须提交)
|
||||
1. 读用户输入。当关键项(文章类型、目标字数、目标读者)缺失或模糊时,调用 `ask_clarification` 向用户提问——**优先给出 `options` 选项**让用户点选(例如目标字数给「800 / 1500 / 3000」,读者给「普通读者 / 行业专家 / 政策决策者」)。一次可就 1-2 个独立的关键项各发一个 ask_clarification。
|
||||
2. 与用户的澄清来回**最多 2 次**。一旦关键信息明确、或已经问过 2 次,就**立即进入第 3 步提交**,缺失字段用合理默认值,绝不无休止追问。
|
||||
3. 调用 `setup_writing` 提交配置:
|
||||
- `user_intent`:精炼成一句话的写作目标(必填)。
|
||||
- `article_type`:见下方可选类型;不确定就留空。
|
||||
- `word_count_target`:正整数;用户没明说就给合理默认(如 800)。
|
||||
- `target_audience`:如「普通读者」「行业专家」;没提就填「普通读者」。
|
||||
- `writing_mode`:用户强调「严格依据素材/不要编造」→ `strict`,否则 `loose`。
|
||||
- `material_source`:**默认 `knowledge_base`(知识库)**;只有当用户明确要求用「技能/技能检索」时才填 `skill`,否则一律 `knowledge_base`。
|
||||
- `outline`:用户提供的大纲原文(保留其层级与标题),否则留空。
|
||||
4. 调用 `setup_writing` 前,用**一句话**说明你将提交的配置要点;调用之后不要再输出任何文章内容。
|
||||
|
||||
{type_block}
|
||||
"""
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
@@ -996,16 +1036,16 @@ Writing mode is enabled for this run.
|
||||
network_policy_section = _build_network_policy_section()
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
|
||||
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
|
||||
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
|
||||
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
|
||||
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
|
||||
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
|
||||
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
|
||||
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
|
||||
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
+4
@@ -67,6 +67,10 @@ class SetupAgentApprovalMiddleware(AgentMiddleware[SetupAgentApprovalMiddlewareS
|
||||
|
||||
args = setup_call.get("args") or {}
|
||||
tool_call_id = setup_call.get("id") or ""
|
||||
|
||||
if isinstance(args.get("skills"), list) and len(args["skills"]) > 5:
|
||||
args = {**args, "skills": args["skills"][:5]}
|
||||
|
||||
payload = self._build_payload(args)
|
||||
logger.info("Setup agent approval requested before writing agent files")
|
||||
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
"""Approval middleware for the AI-writing config setup tool.
|
||||
|
||||
与 ``SetupAgentApprovalMiddleware`` 同构:模型在「写作配置 Q&A」里调用 ``setup_writing``
|
||||
时,于 ``after_model`` 拦截,附上一条结构化审批 ToolMessage(建议的写作配置打包在
|
||||
``additional_kwargs["setup_writing_approval"]``)并 ``jump_to="end"`` 立即结束本轮运行——
|
||||
前端据此渲染审批卡,用户点「应用到表单」后把这些字段回填到左侧写作表单。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from hashlib import sha256
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware, hook_config
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SetupWritingApprovalMiddlewareState(AgentState):
|
||||
"""Compatible with the DeerFlow thread state schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SetupWritingApprovalMiddleware(AgentMiddleware[SetupWritingApprovalMiddlewareState]):
|
||||
"""Stop before ``setup_writing`` and ask the user to apply the proposed config.
|
||||
|
||||
See ``SetupAgentApprovalMiddleware`` for the rationale behind intercepting in
|
||||
``after_model`` + ``jump_to="end"`` (robust against parallel tool calls).
|
||||
"""
|
||||
|
||||
state_schema = SetupWritingApprovalMiddlewareState
|
||||
|
||||
def _is_approved(self, runtime: Runtime) -> bool:
|
||||
context = getattr(runtime, "context", None) or {}
|
||||
if not isinstance(context, dict):
|
||||
return False
|
||||
return context.get("setup_writing_approved") is True
|
||||
|
||||
def _stable_message_id(self, tool_call_id: str, args: dict[str, Any]) -> str:
|
||||
if tool_call_id:
|
||||
return f"setup-writing-approval:{tool_call_id}"
|
||||
digest = sha256(repr(sorted(args.items())).encode("utf-8")).hexdigest()[:16]
|
||||
return f"setup-writing-approval:{digest}"
|
||||
|
||||
def _coerce_word_count(self, raw: Any) -> int | None:
|
||||
if isinstance(raw, bool): # bool 是 int 子类,先排除
|
||||
return None
|
||||
if isinstance(raw, int):
|
||||
return raw if raw > 0 else None
|
||||
if isinstance(raw, float):
|
||||
return int(raw) if raw > 0 else None
|
||||
if isinstance(raw, str):
|
||||
import re
|
||||
|
||||
m = re.search(r"\d[\d,]*", raw)
|
||||
if m:
|
||||
try:
|
||||
value = int(m.group().replace(",", ""))
|
||||
return value if value > 0 else None
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _build_payload(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||
# 素材来源:仅当模型明确给出 "skill" 时才用技能检索;用户没明确要求就一律默认知识库。
|
||||
material_source = (
|
||||
"skill" if str(args.get("material_source") or "").strip() == "skill" else "knowledge_base"
|
||||
)
|
||||
return {
|
||||
"user_intent": str(args.get("user_intent") or "").strip(),
|
||||
"article_type": str(args.get("article_type") or "").strip(),
|
||||
"word_count_target": self._coerce_word_count(args.get("word_count_target")),
|
||||
"target_audience": str(args.get("target_audience") or "").strip(),
|
||||
"writing_mode": str(args.get("writing_mode") or "").strip(),
|
||||
"material_source": material_source,
|
||||
"outline": str(args.get("outline") or "").strip(),
|
||||
}
|
||||
|
||||
def _intercept(self, last: AIMessage) -> dict[str, Any] | None:
|
||||
"""Return a state update if setup_writing must be gated; None to passthrough."""
|
||||
tool_calls = list(last.tool_calls or [])
|
||||
setup_call = next((tc for tc in tool_calls if tc.get("name") == "setup_writing"), None)
|
||||
if setup_call is None:
|
||||
return None
|
||||
|
||||
args = setup_call.get("args") or {}
|
||||
tool_call_id = setup_call.get("id") or ""
|
||||
payload = self._build_payload(args)
|
||||
logger.info("Setup writing approval requested before applying writing config")
|
||||
|
||||
approval_msg = ToolMessage(
|
||||
id=self._stable_message_id(tool_call_id, args),
|
||||
content="请确认是否把这份写作配置应用到表单。",
|
||||
tool_call_id=tool_call_id,
|
||||
name="setup_writing",
|
||||
additional_kwargs={"setup_writing_approval": payload},
|
||||
)
|
||||
|
||||
return {
|
||||
"messages": [approval_msg],
|
||||
"jump_to": "end",
|
||||
}
|
||||
|
||||
@hook_config(can_jump_to=["end"])
|
||||
@override
|
||||
def after_model(
|
||||
self,
|
||||
state: SetupWritingApprovalMiddlewareState,
|
||||
runtime: Runtime,
|
||||
) -> dict[str, Any] | None:
|
||||
if self._is_approved(runtime):
|
||||
return None
|
||||
messages = state.get("messages") or []
|
||||
if not messages:
|
||||
return None
|
||||
last = messages[-1]
|
||||
if not isinstance(last, AIMessage):
|
||||
return None
|
||||
return self._intercept(last)
|
||||
|
||||
@hook_config(can_jump_to=["end"])
|
||||
@override
|
||||
async def aafter_model(
|
||||
self,
|
||||
state: SetupWritingApprovalMiddlewareState,
|
||||
runtime: Runtime,
|
||||
) -> dict[str, Any] | None:
|
||||
return self.after_model(state, runtime)
|
||||
+50
-6
@@ -23,6 +23,7 @@ class TitleMiddlewareState(AgentState):
|
||||
"""Compatible with the `ThreadState` schema."""
|
||||
|
||||
title: NotRequired[str | None]
|
||||
title_provisional: NotRequired[bool]
|
||||
|
||||
|
||||
class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
@@ -61,16 +62,37 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
|
||||
return ""
|
||||
|
||||
def _should_generate_title(self, state: TitleMiddlewareState) -> bool:
|
||||
"""Check if we should generate a title for this thread."""
|
||||
def _should_set_provisional_title(self, state: TitleMiddlewareState) -> bool:
|
||||
"""Check if we should set an instant, question-derived placeholder title.
|
||||
|
||||
Runs in ``before_model`` so the thread shows a sensible title the moment
|
||||
the user sends their first message — eliminating the "untitled" window
|
||||
while the (possibly long) first answer streams. No LLM call, no latency.
|
||||
"""
|
||||
config = self._get_title_config()
|
||||
if not config.enabled:
|
||||
return False
|
||||
|
||||
# Check if thread already has a title in state
|
||||
# Already has a title (provisional or final) — nothing to do.
|
||||
if state.get("title"):
|
||||
return False
|
||||
|
||||
# Only on the very first turn, as soon as the user message arrives.
|
||||
messages = state.get("messages", [])
|
||||
user_messages = [m for m in messages if m.type == "human"]
|
||||
return len(user_messages) == 1
|
||||
|
||||
def _should_generate_title(self, state: TitleMiddlewareState) -> bool:
|
||||
"""Check if we should generate the final LLM title for this thread."""
|
||||
config = self._get_title_config()
|
||||
if not config.enabled:
|
||||
return False
|
||||
|
||||
# Skip only if we already have a *finalized* title. A provisional
|
||||
# placeholder set in before_model should still be upgraded by the LLM.
|
||||
if state.get("title") and not state.get("title_provisional"):
|
||||
return False
|
||||
|
||||
# Check if this is the first turn (has at least one user message and one assistant response)
|
||||
messages = state.get("messages", [])
|
||||
if len(messages) < 2:
|
||||
@@ -146,13 +168,27 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
config["tags"] = [*(config.get("tags") or []), "middleware:title"]
|
||||
return config
|
||||
|
||||
def _provisional_title_result(self, state: TitleMiddlewareState) -> dict | None:
|
||||
"""Set an instant placeholder title from the first user message (no LLM)."""
|
||||
if not self._should_set_provisional_title(state):
|
||||
return None
|
||||
|
||||
messages = state.get("messages", [])
|
||||
user_msg_content = next((m.content for m in messages if m.type == "human"), "")
|
||||
user_msg = self._normalize_content(user_msg_content)
|
||||
if not user_msg.strip():
|
||||
return None
|
||||
title = self._fallback_title(user_msg)
|
||||
logger.debug("TitleMiddleware: set provisional title %r before first model call", title)
|
||||
return {"title": title, "title_provisional": True}
|
||||
|
||||
def _generate_title_result(self, state: TitleMiddlewareState) -> dict | None:
|
||||
"""Generate a local fallback title without blocking on an LLM call."""
|
||||
if not self._should_generate_title(state):
|
||||
return None
|
||||
|
||||
_, user_msg = self._build_title_prompt(state)
|
||||
return {"title": self._fallback_title(user_msg)}
|
||||
return {"title": self._fallback_title(user_msg), "title_provisional": False}
|
||||
|
||||
def _force_disable_thinking(self, model: object) -> None:
|
||||
"""Force-disable thinking on the title model via `extra_body` overrides.
|
||||
@@ -204,10 +240,18 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
response = await model.ainvoke(prompt, config=self._get_runnable_config())
|
||||
title = self._parse_title(response.content)
|
||||
if title:
|
||||
return {"title": title}
|
||||
return {"title": title, "title_provisional": False}
|
||||
except Exception:
|
||||
logger.debug("Failed to generate async title; falling back to local title", exc_info=True)
|
||||
return {"title": self._fallback_title(user_msg)}
|
||||
return {"title": self._fallback_title(user_msg), "title_provisional": False}
|
||||
|
||||
@override
|
||||
def before_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
return self._provisional_title_result(state)
|
||||
|
||||
@override
|
||||
async def abefore_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
return self._provisional_title_result(state)
|
||||
|
||||
@override
|
||||
def after_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
|
||||
+16
@@ -18,6 +18,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_OUTLINE_PREVIEW_LINES = 5
|
||||
|
||||
# Image files are understood via the `view_image` tool (which routes to the
|
||||
# configured vision model), never by reading their bytes. Listing them with the
|
||||
# generic read_file/grep guidance made the agent try to parse image binaries
|
||||
# with bash/python, which fails and wastes turns.
|
||||
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
|
||||
|
||||
|
||||
def _extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]:
|
||||
"""Return the document outline and fallback preview for *file_path*.
|
||||
@@ -88,6 +94,16 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
|
||||
size_str = f"{size_kb:.1f} KB" if size_kb < 1024 else f"{size_kb / 1024:.1f} MB"
|
||||
lines.append(f"- {file['filename']} ({size_str})")
|
||||
lines.append(f" Path: {file['path']}")
|
||||
|
||||
# Images must be understood through the vision model, not by reading bytes.
|
||||
extension = str(file.get("extension") or Path(file["filename"]).suffix).lower()
|
||||
if extension in _IMAGE_EXTENSIONS:
|
||||
lines.append(f" This is an image. To understand it, call the `view_image` tool with image_path='{file['path']}' (optionally pass a `question`).")
|
||||
lines.append(" The configured vision model will analyze it and return a description — trust that result.")
|
||||
lines.append(" Do NOT read, open, or parse the image with read_file, grep, bash, python, PIL, or any byte-level inspection; those cannot interpret image content.")
|
||||
lines.append("")
|
||||
return
|
||||
|
||||
outline = file.get("outline") or []
|
||||
if outline:
|
||||
truncated = outline[-1].get("truncated", False)
|
||||
|
||||
+6
@@ -139,6 +139,12 @@ class ViewImageMiddleware(AgentMiddleware[ViewImageMiddlewareState]):
|
||||
if not messages:
|
||||
return False
|
||||
|
||||
# Nothing to inject when no image data was captured (e.g. delegated
|
||||
# recognition handled the image in the tool and stored no base64).
|
||||
# Avoids injecting a useless "No images have been viewed." message.
|
||||
if not state.get("viewed_images"):
|
||||
return False
|
||||
|
||||
# Get the last assistant message
|
||||
last_assistant_msg = self._get_last_assistant_message(messages)
|
||||
if not last_assistant_msg:
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
"""写作配置 Q&A 的「轮数上限」中间件。
|
||||
|
||||
意图阶段允许与用户来回澄清确认,但**最多 N 轮后必须出结果**:到上限时,在模型请求里
|
||||
注入强制指令 + 用 ``tool_choice`` 锁定 ``setup_writing``,逼模型立刻提交配置卡,不再提问。
|
||||
|
||||
「一轮」= 一条用户消息(HumanMessage)。达到 ``max_rounds`` 条用户消息时强制提交。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
|
||||
class WritingSetupRoundCapMiddleware(AgentMiddleware[AgentState]):
|
||||
"""达到澄清轮数上限时,强制模型调用 ``setup_writing`` 提交。
|
||||
|
||||
软约束(注入 SystemMessage)+ 硬约束(``tool_choice`` 锁定工具)双保险:即使个别模型
|
||||
无视提示,``tool_choice`` 也会逼它产出一次 ``setup_writing`` 调用。
|
||||
"""
|
||||
|
||||
def __init__(self, max_rounds: int = 3, tool_name: str = "setup_writing") -> None:
|
||||
super().__init__()
|
||||
self._max_rounds = max(1, max_rounds)
|
||||
self._tool_name = tool_name
|
||||
|
||||
def _should_force(self, messages: list) -> bool:
|
||||
human_count = sum(1 for m in messages if isinstance(m, HumanMessage))
|
||||
return human_count >= self._max_rounds
|
||||
|
||||
def _reminder(self) -> SystemMessage:
|
||||
return SystemMessage(
|
||||
content=(
|
||||
f"【系统强制】意图澄清已达 {self._max_rounds} 轮上限。"
|
||||
"现在必须立即调用 setup_writing 提交写作配置,禁止再向用户提任何问题。"
|
||||
"缺失或不确定的字段一律用合理默认值:"
|
||||
"word_count_target 默认 800、target_audience 默认「普通读者」、"
|
||||
"writing_mode 默认 loose、material_source 默认 knowledge_base、"
|
||||
"article_type 不确定就留空。"
|
||||
)
|
||||
)
|
||||
|
||||
def _patch(self, request: ModelRequest) -> ModelRequest:
|
||||
return request.override(
|
||||
messages=[*request.messages, self._reminder()],
|
||||
tool_choice=self._tool_name,
|
||||
)
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
if self._should_force(request.messages):
|
||||
request = self._patch(request)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
if self._should_force(request.messages):
|
||||
request = self._patch(request)
|
||||
return await handler(request)
|
||||
@@ -69,6 +69,10 @@ class ThreadState(AgentState):
|
||||
sandbox: NotRequired[SandboxState | None]
|
||||
thread_data: NotRequired[ThreadDataState | None]
|
||||
title: NotRequired[str | None]
|
||||
# True while `title` is a provisional, question-derived placeholder that the
|
||||
# async LLM pass has not yet finalized. Lets TitleMiddleware show a title
|
||||
# instantly (no "untitled" window) and upgrade it once the answer is ready.
|
||||
title_provisional: NotRequired[bool]
|
||||
artifacts: Annotated[list[str], merge_artifacts]
|
||||
todos: NotRequired[list | None]
|
||||
uploaded_files: NotRequired[list[dict] | None]
|
||||
|
||||
@@ -15,6 +15,7 @@ from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.login_config import AuthLoginConfig
|
||||
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
|
||||
from deerflow.config.model_config import ModelConfig
|
||||
from deerflow.config.run_events_config import RunEventsConfig
|
||||
@@ -32,6 +33,7 @@ from deerflow.config.title_config import TitleConfig, load_title_config_from_dic
|
||||
from deerflow.config.token_usage_config import TokenUsageConfig
|
||||
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
|
||||
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
|
||||
from deerflow.config.vision_config import VisionConfig
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@@ -104,6 +106,8 @@ class AppConfig(BaseModel):
|
||||
log_level: str = Field(default="info", description="Logging level for deerflow and app modules (debug/info/warning/error); third-party libraries are not affected")
|
||||
token_usage: TokenUsageConfig = Field(default_factory=TokenUsageConfig, description="Token usage tracking configuration")
|
||||
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
|
||||
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
|
||||
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
|
||||
sandbox: SandboxConfig = Field(description="Sandbox configuration")
|
||||
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
|
||||
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
|
||||
|
||||
+51
@@ -62,6 +62,52 @@ class SkillDisplayTableColumn(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayResultSetConfig(BaseModel):
|
||||
"""Display adapter for one result array inside a skill response."""
|
||||
|
||||
key: str = Field(description="Stable key for this result set")
|
||||
label: str | None = Field(default=None, description="Human-readable result set label")
|
||||
enabled: bool = Field(default=True, description="Whether this result set is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="citation",
|
||||
description="How the frontend should render this result set",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="singleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayConfig(BaseModel):
|
||||
"""Frontend display adapter for structured skill results."""
|
||||
|
||||
@@ -103,6 +149,11 @@ class SkillDisplayConfig(BaseModel):
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
result_sets: list[SkillDisplayResultSetConfig] = Field(
|
||||
default_factory=list,
|
||||
description="Optional per-result-array display adapters",
|
||||
alias="resultSets",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Fallback shared password for the username-login gate. Used when
|
||||
# ``UsernameLoginConfig.password`` is left empty but ``require_password`` is on.
|
||||
DEFAULT_USERNAME_LOGIN_PASSWORD = "123ewq"
|
||||
|
||||
|
||||
class UsernameLoginConfig(BaseModel):
|
||||
"""Gate for the passwordless ``POST /api/v1/auth/login/username`` flow.
|
||||
|
||||
By default this flow logs a user in (auto-registering if needed) from a
|
||||
username alone — convenient, but it lets anyone sign in as any username.
|
||||
Turning ``require_password`` on forces callers to additionally pass a
|
||||
shared secret as ``?password=...``; mismatches are rejected with 401.
|
||||
"""
|
||||
|
||||
require_password: bool = Field(
|
||||
default=False,
|
||||
description="When true, login/username requires a matching ?password= query value.",
|
||||
)
|
||||
password: str = Field(
|
||||
default="",
|
||||
description=f"Shared secret for the username-login gate. Empty falls back to '{DEFAULT_USERNAME_LOGIN_PASSWORD}'.",
|
||||
)
|
||||
|
||||
@property
|
||||
def effective_password(self) -> str:
|
||||
"""The password actually checked at login time (config value or default)."""
|
||||
return self.password or DEFAULT_USERNAME_LOGIN_PASSWORD
|
||||
|
||||
|
||||
class TokenLoginConfig(BaseModel):
|
||||
"""Token-exchange login.
|
||||
|
||||
The frontend passes an upstream access token via the URL. The gateway calls
|
||||
``token_info_url`` with that token placed directly in the
|
||||
``Authorization: Bearer <token>`` header, reads ``data.username`` from the
|
||||
response, then signs the matching local user in (auto-registering if the
|
||||
username is not yet in the database).
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Master switch for the POST /api/v1/auth/login/token endpoint.",
|
||||
)
|
||||
token_info_url: str = Field(
|
||||
default="",
|
||||
description="Upstream endpoint that resolves an access token to a username.",
|
||||
)
|
||||
service_authorization: str = Field(
|
||||
default="Bearer admin",
|
||||
description=(
|
||||
"Deprecated/unused: the Authorization header is now built from the "
|
||||
"caller's token as 'Bearer <token>'. Retained for backward compat."
|
||||
),
|
||||
)
|
||||
timeout_seconds: float = Field(
|
||||
default=10.0,
|
||||
gt=0,
|
||||
description="HTTP timeout (seconds) for the token_info_url call.",
|
||||
)
|
||||
verify_ssl: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Whether to verify the upstream TLS certificate. Set false for "
|
||||
"internal/UAT hosts with self-signed or private-CA certs. Leave "
|
||||
"true in production."
|
||||
),
|
||||
)
|
||||
ca_cert_path: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Optional path to a CA bundle (PEM) used to verify the upstream "
|
||||
"certificate. Takes precedence over verify_ssl when set."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class AuthLoginConfig(BaseModel):
|
||||
"""Container for the custom login modes layered on top of the base auth."""
|
||||
|
||||
username_login: UsernameLoginConfig = Field(default_factory=UsernameLoginConfig)
|
||||
token_login: TokenLoginConfig = Field(default_factory=TokenLoginConfig)
|
||||
@@ -0,0 +1,29 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Default instruction sent to the vision model when the agent does not provide a
|
||||
# more specific question. Kept in Chinese to match the product's primary locale.
|
||||
DEFAULT_VISION_PROMPT = "请仔细观察这张图片,尽可能详细、准确地描述其中的全部内容,包括文字、图表、物体、场景与布局等关键信息。"
|
||||
|
||||
|
||||
class VisionConfig(BaseModel):
|
||||
"""Configuration for image recognition (识图).
|
||||
|
||||
When ``model_name`` names a model from ``models[]``, the lead agent's
|
||||
``view_image`` tool delegates image understanding to that dedicated model
|
||||
and returns a text description — so a text-only chat model can still "see"
|
||||
uploaded images. When ``model_name`` is unset, vision falls back to the main
|
||||
chat model, which only works when that model has ``supports_vision: true``.
|
||||
"""
|
||||
|
||||
model_name: str | None = Field(
|
||||
default=None,
|
||||
description="Name of the model (from models[]) used for image recognition. When unset, falls back to the main chat model if it supports vision.",
|
||||
)
|
||||
prompt: str = Field(
|
||||
default=DEFAULT_VISION_PROMPT,
|
||||
description="Default instruction for the vision model when no per-call question is given.",
|
||||
)
|
||||
max_tokens: int | None = Field(
|
||||
default=None,
|
||||
description="Optional override for the vision model's max output tokens; falls back to the model's own config when unset.",
|
||||
)
|
||||
@@ -1,3 +1,10 @@
|
||||
from .factory import create_chat_model
|
||||
from .vision import is_delegated_vision, recognize_image, resolve_vision_model_name, vision_enabled
|
||||
|
||||
__all__ = ["create_chat_model"]
|
||||
__all__ = [
|
||||
"create_chat_model",
|
||||
"is_delegated_vision",
|
||||
"recognize_image",
|
||||
"resolve_vision_model_name",
|
||||
"vision_enabled",
|
||||
]
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Image recognition (识图) via a configurable vision model.
|
||||
|
||||
When ``config.yaml`` defines ``vision.model_name``, the lead agent's
|
||||
``view_image`` tool delegates image understanding to that dedicated model and
|
||||
returns a text description — so a text-only chat model can still "see" uploaded
|
||||
images. When no dedicated model is configured, vision falls back to the main
|
||||
chat model, which only works when that model has ``supports_vision: true``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.vision_config import DEFAULT_VISION_PROMPT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _configured_vision_model(config: AppConfig) -> str | None:
|
||||
"""Return a valid dedicated vision model name from config, or None."""
|
||||
vision_cfg = getattr(config, "vision", None)
|
||||
name = getattr(vision_cfg, "model_name", None) if vision_cfg else None
|
||||
if not name:
|
||||
return None
|
||||
if config.get_model_config(name) is None:
|
||||
logger.warning("vision.model_name %r is not defined in models[]; ignoring", name)
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def resolve_vision_model_name(model_name: str | None, *, app_config: AppConfig | None = None) -> str | None:
|
||||
"""Resolve which model should handle image recognition for this run.
|
||||
|
||||
Priority:
|
||||
1. A dedicated ``vision.model_name`` configured in ``config.yaml``.
|
||||
2. The main chat model, only if it declares ``supports_vision: true``.
|
||||
3. ``None`` — vision is unavailable.
|
||||
"""
|
||||
config = app_config or get_app_config()
|
||||
configured = _configured_vision_model(config)
|
||||
if configured:
|
||||
return configured
|
||||
if model_name:
|
||||
mc = config.get_model_config(model_name)
|
||||
if mc is not None and mc.supports_vision:
|
||||
return model_name
|
||||
return None
|
||||
|
||||
|
||||
def vision_enabled(model_name: str | None, *, app_config: AppConfig | None = None) -> bool:
|
||||
"""Whether the ``view_image`` tool/middleware should be available this run."""
|
||||
return resolve_vision_model_name(model_name, app_config=app_config) is not None
|
||||
|
||||
|
||||
def is_delegated_vision(*, app_config: AppConfig | None = None) -> bool:
|
||||
"""Whether image recognition is delegated to a dedicated vision model.
|
||||
|
||||
True when ``vision.model_name`` is configured and valid — in that case
|
||||
``view_image`` calls the dedicated model and returns a text description
|
||||
instead of injecting raw image data into the main chat model.
|
||||
"""
|
||||
config = app_config or get_app_config()
|
||||
return _configured_vision_model(config) is not None
|
||||
|
||||
|
||||
def _extract_text(message) -> str:
|
||||
"""Flatten an LLM response message into plain text."""
|
||||
content = getattr(message, "content", message)
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text", "")))
|
||||
return "\n".join(p for p in parts if p).strip()
|
||||
return str(content).strip()
|
||||
|
||||
|
||||
def recognize_image(
|
||||
image_base64: str,
|
||||
mime_type: str,
|
||||
*,
|
||||
question: str | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> str:
|
||||
"""Describe/analyze an image using the configured dedicated vision model.
|
||||
|
||||
Returns the model's text answer. Raises ``ValueError`` if no dedicated
|
||||
vision model is configured, and propagates model/network errors to the
|
||||
caller for handling.
|
||||
"""
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from deerflow.models.factory import create_chat_model
|
||||
|
||||
config = app_config or get_app_config()
|
||||
model_name = _configured_vision_model(config)
|
||||
if model_name is None:
|
||||
raise ValueError("No dedicated vision model is configured (vision.model_name).")
|
||||
|
||||
vision_cfg = getattr(config, "vision", None)
|
||||
default_prompt = getattr(vision_cfg, "prompt", None) or DEFAULT_VISION_PROMPT
|
||||
prompt = (question or "").strip() or default_prompt
|
||||
|
||||
extra: dict = {}
|
||||
max_tokens = getattr(vision_cfg, "max_tokens", None) if vision_cfg else None
|
||||
if max_tokens:
|
||||
extra["max_tokens"] = max_tokens
|
||||
|
||||
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config, **extra)
|
||||
message = HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_base64}"}},
|
||||
]
|
||||
)
|
||||
result = model.invoke([message])
|
||||
return _extract_text(result)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Persistence for iframe embed session → thread mappings."""
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.memory import MemoryEmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.sql import EmbedSessionRepository
|
||||
|
||||
__all__ = [
|
||||
"EmbedSessionStore",
|
||||
"MemoryEmbedSessionStore",
|
||||
"EmbedSessionRepository",
|
||||
"make_embed_session_store",
|
||||
]
|
||||
|
||||
|
||||
def make_embed_session_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryEmbedSessionStore()
|
||||
return EmbedSessionRepository(session_factory)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Abstract persistence API for iframe embed session → thread mappings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class EmbedSessionStore(ABC):
|
||||
@abstractmethod
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
"""Create or update the mapping for an embed session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
"""Return ``{session_id, thread_id, created_at, updated_at}`` or None."""
|
||||
raise NotImplementedError
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""In-memory ``EmbedSessionStore`` for tests / memory backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
|
||||
|
||||
class MemoryEmbedSessionStore(EmbedSessionStore):
|
||||
def __init__(self) -> None:
|
||||
self._sessions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
existing = self._sessions.get(session_id)
|
||||
if existing is None:
|
||||
record = {
|
||||
"session_id": session_id,
|
||||
"thread_id": thread_id,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
else:
|
||||
record = {
|
||||
**existing,
|
||||
"thread_id": thread_id,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._sessions[session_id] = record
|
||||
return copy.deepcopy(record)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
record = self._sessions.get(session_id)
|
||||
return copy.deepcopy(record) if record is not None else None
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""ORM model for iframe embed session → thread lookups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class EmbedSessionRow(Base):
|
||||
__tablename__ = "embed_sessions"
|
||||
|
||||
# Opaque id generated by the parent page and passed into the iframe URL.
|
||||
session_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""SQLAlchemy-backed embed session repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
|
||||
|
||||
|
||||
def _row_to_dict(row: EmbedSessionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class EmbedSessionRepository(EmbedSessionStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(EmbedSessionRow, session_id)
|
||||
now = datetime.now(UTC)
|
||||
if row is None:
|
||||
row = EmbedSessionRow(
|
||||
session_id=session_id,
|
||||
thread_id=thread_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.thread_id = thread_id
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(EmbedSessionRow, session_id)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Generated HTML page + favorite persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
from deerflow.persistence.html_pages.memory import MemoryHtmlPageStore
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
from deerflow.persistence.html_pages.sql import HtmlPageRepository
|
||||
|
||||
__all__ = [
|
||||
"HtmlPageStore",
|
||||
"HtmlPageRepository",
|
||||
"MemoryHtmlPageStore",
|
||||
"HtmlPageFavoriteRow",
|
||||
"ScheduledHtmlPageRow",
|
||||
"make_html_page_store",
|
||||
]
|
||||
|
||||
|
||||
def make_html_page_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryHtmlPageStore()
|
||||
return HtmlPageRepository(session_factory)
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Abstract persistence API for generated HTML pages and favorites."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class HtmlPageStore(ABC):
|
||||
# --- Generated pages ---
|
||||
|
||||
@abstractmethod
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
"""Return a page by id without restricting by owner."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Return the page produced by a given run, if any."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Return the most recently created page for a task, if any."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
@abstractmethod
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
"""Return a favorite by id without restricting by owner."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
"""Return the user's existing favorite for a page, if any (dedup check)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""In-memory HtmlPageStore for no-database / test deployments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryHtmlPageStore(HtmlPageStore):
|
||||
def __init__(self) -> None:
|
||||
self._pages: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# --- Generated pages ---
|
||||
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"task_id": data["task_id"],
|
||||
"run_id": data["run_id"],
|
||||
"user_id": data["user_id"],
|
||||
"title": data.get("title") or "",
|
||||
"html_content": data.get("html_content") or "",
|
||||
"html_file_path": data.get("html_file_path"),
|
||||
"data_snapshot": data.get("data_snapshot") or {},
|
||||
"prompt_snapshot": data.get("prompt_snapshot") or {},
|
||||
"reference_favorite_id": data.get("reference_favorite_id"),
|
||||
"style_summary": data.get("style_summary"),
|
||||
"layout_summary": data.get("layout_summary"),
|
||||
"created_at": _now_iso(),
|
||||
"updated_at": _now_iso(),
|
||||
}
|
||||
self._pages[row["id"]] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
row = self._pages.get(page_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
matches = [r for r in self._pages.values() if r["run_id"] == run_id]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
matches = [r for r in self._pages.values() if r["task_id"] == task_id]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"user_id": data["user_id"],
|
||||
"page_id": data["page_id"],
|
||||
"title": data.get("title") or "",
|
||||
"description": data.get("description"),
|
||||
"tags": data.get("tags") or [],
|
||||
"html_snapshot": data.get("html_snapshot") or "",
|
||||
"style_summary": data.get("style_summary"),
|
||||
"layout_summary": data.get("layout_summary"),
|
||||
"content_structure_summary": data.get("content_structure_summary"),
|
||||
"reuse_prompt": data.get("reuse_prompt"),
|
||||
"created_at": _now_iso(),
|
||||
"updated_at": _now_iso(),
|
||||
}
|
||||
self._favorites[row["id"]] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
matches = [r for r in self._favorites.values() if r["user_id"] == user_id]
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return [copy.deepcopy(r) for r in matches[: max(1, limit)]]
|
||||
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
row = self._favorites.get(favorite_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
for row in self._favorites.values():
|
||||
if row["user_id"] == user_id and row["page_id"] == page_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
row = self._favorites.get(favorite_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return False
|
||||
del self._favorites[favorite_id]
|
||||
return True
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""ORM models for generated HTML pages and user favorites.
|
||||
|
||||
These back the "HTML 页面定时任务" feature: a scheduled task of kind
|
||||
``html_page`` renders a full HTML document each run. Every rendered page is
|
||||
recorded in :class:`ScheduledHtmlPageRow`; a user may bookmark one into
|
||||
:class:`HtmlPageFavoriteRow`, which also stores a reusable style/layout
|
||||
summary so a later task can be generated "in the style of" the favorite.
|
||||
|
||||
Both tables are brand new, so ``Base.metadata.create_all`` creates them on the
|
||||
next startup without touching any existing schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class ScheduledHtmlPageRow(Base):
|
||||
"""One HTML page produced by a single run of an ``html_page`` task."""
|
||||
|
||||
__tablename__ = "scheduled_html_pages"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
# Full HTML document — can be large, so LONGTEXT on MySQL.
|
||||
html_content: Mapped[str] = mapped_column(PortableLongText())
|
||||
html_file_path: Mapped[str | None] = mapped_column(Text)
|
||||
data_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
prompt_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
reference_favorite_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
style_summary: Mapped[str | None] = mapped_column(Text)
|
||||
layout_summary: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class HtmlPageFavoriteRow(Base):
|
||||
"""A user-bookmarked HTML page, with a reusable style/layout summary."""
|
||||
|
||||
__tablename__ = "html_page_favorites"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
page_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
tags_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list)
|
||||
# Snapshot of the page HTML at favorite time, decoupled from the source run
|
||||
# so deleting the run never breaks the favorite reference.
|
||||
html_snapshot: Mapped[str] = mapped_column(PortableLongText())
|
||||
style_summary: Mapped[str | None] = mapped_column(Text)
|
||||
layout_summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_structure_summary: Mapped[str | None] = mapped_column(Text)
|
||||
reuse_prompt: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
"""SQLAlchemy repository for generated HTML pages and favorites."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
|
||||
|
||||
def _to_dict(row) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
# Normalize the JSON column names to API-friendly keys.
|
||||
if "data_snapshot_json" in data:
|
||||
data["data_snapshot"] = data.pop("data_snapshot_json") or {}
|
||||
if "prompt_snapshot_json" in data:
|
||||
data["prompt_snapshot"] = data.pop("prompt_snapshot_json") or {}
|
||||
if "tags_json" in data:
|
||||
data["tags"] = data.pop("tags_json") or []
|
||||
return data
|
||||
|
||||
|
||||
class HtmlPageRepository(HtmlPageStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# --- Generated pages ---
|
||||
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = ScheduledHtmlPageRow(
|
||||
id=data["id"],
|
||||
task_id=data["task_id"],
|
||||
run_id=data["run_id"],
|
||||
user_id=data["user_id"],
|
||||
title=data.get("title") or "",
|
||||
html_content=data.get("html_content") or "",
|
||||
html_file_path=data.get("html_file_path"),
|
||||
data_snapshot_json=data.get("data_snapshot") or {},
|
||||
prompt_snapshot_json=data.get("prompt_snapshot") or {},
|
||||
reference_favorite_id=data.get("reference_favorite_id"),
|
||||
style_summary=data.get("style_summary"),
|
||||
layout_summary=data.get("layout_summary"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledHtmlPageRow, page_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledHtmlPageRow)
|
||||
.where(ScheduledHtmlPageRow.run_id == run_id)
|
||||
.order_by(ScheduledHtmlPageRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledHtmlPageRow)
|
||||
.where(ScheduledHtmlPageRow.task_id == task_id)
|
||||
.order_by(ScheduledHtmlPageRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = HtmlPageFavoriteRow(
|
||||
id=data["id"],
|
||||
user_id=data["user_id"],
|
||||
page_id=data["page_id"],
|
||||
title=data.get("title") or "",
|
||||
description=data.get("description"),
|
||||
tags_json=data.get("tags") or [],
|
||||
html_snapshot=data.get("html_snapshot") or "",
|
||||
style_summary=data.get("style_summary"),
|
||||
layout_summary=data.get("layout_summary"),
|
||||
content_structure_summary=data.get("content_structure_summary"),
|
||||
reuse_prompt=data.get("reuse_prompt"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(HtmlPageFavoriteRow)
|
||||
.where(HtmlPageFavoriteRow.user_id == user_id)
|
||||
.order_by(HtmlPageFavoriteRow.created_at.desc())
|
||||
.limit(max(1, limit))
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(HtmlPageFavoriteRow, favorite_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(HtmlPageFavoriteRow)
|
||||
.where(HtmlPageFavoriteRow.user_id == user_id, HtmlPageFavoriteRow.page_id == page_id)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(HtmlPageFavoriteRow).where(
|
||||
HtmlPageFavoriteRow.id == favorite_id,
|
||||
HtmlPageFavoriteRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Create ``embed_sessions`` table for iframe embed status polling.
|
||||
|
||||
Revision ID: 20260602_01
|
||||
Revises: 20260528_02
|
||||
Create Date: 2026-06-02
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20260602_01"
|
||||
down_revision = "20260528_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("embed_sessions"):
|
||||
op.create_table(
|
||||
"embed_sessions",
|
||||
sa.Column("session_id", sa.String(128), primary_key=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_embed_sessions_thread_id", "embed_sessions", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("embed_sessions"):
|
||||
op.drop_table("embed_sessions")
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Add step3 column to roundtable_drafts.
|
||||
|
||||
Step 3「结果绘制」persistence: the generated report (HTML + metadata) is stored
|
||||
inline in the draft so a reloaded draft can show the report without depending on
|
||||
the ephemeral report thread. The column is LONGTEXT on MySQL (HTML can exceed the
|
||||
64 KB TEXT cap) and plain TEXT (unbounded) elsewhere — matching ``PortableLongText``
|
||||
on ``RoundtableDraftRow.step3``. ``create_all`` adds the column for brand-new DBs;
|
||||
this revision patches DBs that already have the table.
|
||||
|
||||
Revision ID: 20260603_01
|
||||
Revises: 20260602_01
|
||||
Create Date: 2026-06-03
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
|
||||
|
||||
revision = "20260603_01"
|
||||
down_revision = "20260602_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("roundtable_drafts") and not _has_column("roundtable_drafts", "step3"):
|
||||
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
|
||||
op.add_column("roundtable_drafts", sa.Column("step3", col_type, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("roundtable_drafts", "step3"):
|
||||
op.drop_column("roundtable_drafts", "step3")
|
||||
+10
@@ -22,6 +22,11 @@ from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
|
||||
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||
from deerflow.persistence.roundtable_drafts.model import (
|
||||
RoundtableDraftRow,
|
||||
RoundtableRecommendHistoryRow,
|
||||
)
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
ScheduledTaskDeliveryProfileRow,
|
||||
@@ -33,6 +38,7 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
@@ -45,6 +51,9 @@ __all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"NotificationRow",
|
||||
"RecommendedQuestionRow",
|
||||
"RoundtableChainRow",
|
||||
"RoundtableDraftRow",
|
||||
"RoundtableRecommendHistoryRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
@@ -56,6 +65,7 @@ __all__ = [
|
||||
"SkillDisplayAdapterRow",
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"EmbedSessionRow",
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Roundtable-planning business-chain persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||
from deerflow.persistence.roundtable_chains.sql import RoundtableChainRepository
|
||||
|
||||
__all__ = [
|
||||
"RoundtableChainRepository",
|
||||
"RoundtableChainRow",
|
||||
"make_roundtable_chain_store",
|
||||
]
|
||||
|
||||
|
||||
def make_roundtable_chain_store(session_factory):
|
||||
if session_factory is None:
|
||||
return None
|
||||
return RoundtableChainRepository(session_factory)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""ORM model for roundtable-planning business chains.
|
||||
|
||||
A "business chain" (业务链条) is a named, reusable, *ordered* list of agents the
|
||||
user assembles in the chain editor. In Step 2 the coordinator dispatches to the
|
||||
seats following this order (see ``multi-agent-business-chain-dev.md``).
|
||||
|
||||
Persisted per user (like ``roundtable_drafts``) so it follows the account across
|
||||
devices/sessions. ``seats`` is stored as a JSON array serialized into
|
||||
``PortableLongText`` (MySQL ``LONGTEXT``) rather than ``PortableJSON`` for
|
||||
headroom, mirroring the sibling roundtable persistence modules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class RoundtableChainRow(Base):
|
||||
__tablename__ = "roundtable_chains"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Human-readable chain title shown on the picker card.
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# Optional free-text description.
|
||||
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
# JSON: ordered [{"agent_id": str, "name": str, "description": str}, ...].
|
||||
# seats[0] is the first seat to speak. Stored as LONGTEXT-backed JSON string.
|
||||
seats: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""SQLAlchemy-backed roundtable business-chain store.
|
||||
|
||||
The ``seats`` JSON column is stored as a serialized string in
|
||||
``PortableLongText`` and round-tripped here with ``json.dumps`` / ``json.loads``
|
||||
— same approach as ``roundtable_drafts`` / ``ai_writing_sessions``.
|
||||
|
||||
All write/read paths are scoped by ``user_id`` for ownership isolation; a row
|
||||
that belongs to another user is treated as not found.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class RoundtableChainRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: RoundtableChainRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"description": row.description,
|
||||
"seats": _loads(row.seats) or [],
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
async def list_chains(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(RoundtableChainRow)
|
||||
.where(RoundtableChainRow.user_id == user_id)
|
||||
.order_by(RoundtableChainRow.updated_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_chain(self, chain_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableChainRow, chain_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
return self._to_dict(row)
|
||||
|
||||
async def create_chain(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = RoundtableChainRow(
|
||||
id=data["id"],
|
||||
user_id=user_id,
|
||||
title=(data.get("title") or "").strip(),
|
||||
description=(data.get("description") or None),
|
||||
seats=_dumps(data.get("seats") or []),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def update_chain(
|
||||
self,
|
||||
chain_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title=_UNSET,
|
||||
description=_UNSET,
|
||||
seats=_UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableChainRow, chain_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
if title is not _UNSET and title is not None:
|
||||
row.title = str(title).strip()
|
||||
if description is not _UNSET:
|
||||
row.description = description or None
|
||||
if seats is not _UNSET:
|
||||
row.seats = _dumps(seats)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def delete_chain(self, chain_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableChainRow, chain_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Roundtable-planning draft + recommendation-history persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.roundtable_drafts.model import (
|
||||
RoundtableDraftRow,
|
||||
RoundtableRecommendHistoryRow,
|
||||
)
|
||||
from deerflow.persistence.roundtable_drafts.sql import RoundtableDraftRepository
|
||||
|
||||
__all__ = [
|
||||
"RoundtableDraftRepository",
|
||||
"RoundtableDraftRow",
|
||||
"RoundtableRecommendHistoryRow",
|
||||
"make_roundtable_draft_store",
|
||||
]
|
||||
|
||||
|
||||
def make_roundtable_draft_store(session_factory):
|
||||
if session_factory is None:
|
||||
return None
|
||||
return RoundtableDraftRepository(session_factory)
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
"""ORM models for roundtable-planning drafts and recommendation history.
|
||||
|
||||
A "draft" is one full roundtable-planning session (Step 1 intent + Step 2
|
||||
multi-agent roundtable), persisted per user so it follows the account across
|
||||
devices/sessions — replacing the old browser-only ``localStorage`` storage.
|
||||
|
||||
``step1`` / ``step2`` are stored as JSON serialized into ``PortableLongText``
|
||||
(MySQL ``LONGTEXT``) rather than ``PortableJSON`` (MySQL ``TEXT``, 64 KB cap),
|
||||
because the Step 2 snapshot can carry a long roundtable transcript plus per-seat
|
||||
stub messages that easily exceed 64 KB — mirroring ``ai_writing_sessions``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class RoundtableDraftRow(Base):
|
||||
__tablename__ = "roundtable_drafts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Auto-derived task summary (objective → first user message → timestamp).
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# Furthest step reached when this snapshot was taken (1 | 2 | 3).
|
||||
furthest_step: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
# JSON blobs matching the frontend Step1Snapshot / Step2Snapshot shapes.
|
||||
step1: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
step2: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
# Step 3「结果绘制」snapshot: { html, generatedAt, model, summary? }. The full
|
||||
# report HTML is stored inline so a reloaded draft can show the report without
|
||||
# depending on the ephemeral report thread (LONGTEXT — HTML can exceed 64 KB).
|
||||
step3: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class RoundtableRecommendHistoryRow(Base):
|
||||
"""One agent-recommendation run, bound to a draft/session.
|
||||
|
||||
Lets the recommend dialog show "last recommendation for this session" on
|
||||
open, while still letting the user re-analyze (which appends a new row).
|
||||
"""
|
||||
|
||||
__tablename__ = "roundtable_recommend_history"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
draft_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Denormalized intent objective at recommend time (list display / context).
|
||||
objective: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# How the run ended on the backend: "done" | "fallback" | "asking" | "error".
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="done")
|
||||
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# Natural-language rationale (may be long → LONGTEXT).
|
||||
rationale: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# JSON: [{"agent_id": str, "reason": str}, ...]
|
||||
picks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# JSON snapshot of the candidate pool at recommend time:
|
||||
# [{"agent_id": str, "name": str, "description": str}, ...]
|
||||
candidates: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
"""SQLAlchemy-backed roundtable-planning draft + recommendation-history store.
|
||||
|
||||
JSON columns (``step1`` / ``step2`` / ``picks`` / ``candidates``) are stored as
|
||||
serialized strings in ``PortableLongText`` and round-tripped here with
|
||||
``json.dumps`` / ``json.loads`` — same approach as ``ai_writing_sessions`` — so
|
||||
large Step 2 transcripts never hit MySQL's 64 KB ``TEXT`` cap.
|
||||
|
||||
All write/read paths are scoped by ``user_id`` for ownership isolation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.roundtable_drafts.model import (
|
||||
RoundtableDraftRow,
|
||||
RoundtableRecommendHistoryRow,
|
||||
)
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class RoundtableDraftRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── drafts ────────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _draft_to_meta(row: RoundtableDraftRow) -> dict[str, Any]:
|
||||
"""Lightweight projection for the history dropdown (no big blobs)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"furthest_step": row.furthest_step,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _draft_to_dict(cls, row: RoundtableDraftRow) -> dict[str, Any]:
|
||||
d = cls._draft_to_meta(row)
|
||||
d["step1"] = _loads(row.step1)
|
||||
d["step2"] = _loads(row.step2)
|
||||
d["step3"] = _loads(row.step3)
|
||||
return d
|
||||
|
||||
async def list_drafts(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(RoundtableDraftRow)
|
||||
.where(RoundtableDraftRow.user_id == user_id)
|
||||
.order_by(RoundtableDraftRow.updated_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._draft_to_meta(row) for row in result.scalars()]
|
||||
|
||||
async def get_draft(self, draft_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
return self._draft_to_dict(row)
|
||||
|
||||
async def create_draft(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = RoundtableDraftRow(
|
||||
id=data["id"],
|
||||
user_id=user_id,
|
||||
title=(data.get("title") or "").strip(),
|
||||
furthest_step=int(data.get("furthest_step") or 1),
|
||||
step1=_dumps(data.get("step1")),
|
||||
step2=_dumps(data.get("step2")),
|
||||
step3=_dumps(data.get("step3")),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._draft_to_dict(row)
|
||||
|
||||
async def update_draft(
|
||||
self,
|
||||
draft_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title=_UNSET,
|
||||
furthest_step=_UNSET,
|
||||
step1=_UNSET,
|
||||
step2=_UNSET,
|
||||
step3=_UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
if title is not _UNSET and title is not None:
|
||||
row.title = str(title).strip()
|
||||
if furthest_step is not _UNSET and furthest_step is not None:
|
||||
row.furthest_step = int(furthest_step)
|
||||
if step1 is not _UNSET:
|
||||
row.step1 = _dumps(step1)
|
||||
if step2 is not _UNSET:
|
||||
row.step2 = _dumps(step2)
|
||||
if step3 is not _UNSET:
|
||||
row.step3 = _dumps(step3)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._draft_to_dict(row)
|
||||
|
||||
async def delete_draft(self, draft_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# ── recommendation history ────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _history_to_dict(row: RoundtableRecommendHistoryRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"draft_id": row.draft_id,
|
||||
"objective": row.objective,
|
||||
"status": row.status,
|
||||
"model": row.model,
|
||||
"rationale": row.rationale,
|
||||
"picks": _loads(row.picks) or [],
|
||||
"candidates": _loads(row.candidates) or [],
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
}
|
||||
|
||||
async def list_recommendations(
|
||||
self, draft_id: str, user_id: str, *, limit: int = 20
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(RoundtableRecommendHistoryRow)
|
||||
.where(
|
||||
RoundtableRecommendHistoryRow.draft_id == draft_id,
|
||||
RoundtableRecommendHistoryRow.user_id == user_id,
|
||||
)
|
||||
.order_by(RoundtableRecommendHistoryRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._history_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def add_recommendation(self, user_id: str, draft_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = RoundtableRecommendHistoryRow(
|
||||
id=data["id"],
|
||||
user_id=user_id,
|
||||
draft_id=draft_id,
|
||||
objective=(data.get("objective") or "").strip()[:512],
|
||||
status=(data.get("status") or "done")[:32],
|
||||
model=(data.get("model") or None),
|
||||
rationale=data.get("rationale") or None,
|
||||
picks=_dumps(data.get("picks") or []),
|
||||
candidates=_dumps(data.get("candidates") or []),
|
||||
created_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._history_to_dict(row)
|
||||
@@ -7,7 +7,11 @@ import re
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from deerflow.config.extensions_config import SkillDisplayConfig, get_extensions_config
|
||||
from deerflow.config.extensions_config import (
|
||||
SkillDisplayConfig,
|
||||
SkillDisplayResultSetConfig,
|
||||
get_extensions_config,
|
||||
)
|
||||
|
||||
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
@@ -222,7 +226,11 @@ def adapt_reference_batches_with_displays(
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
if not config.enabled or config.mode == "none":
|
||||
has_enabled_result_set = any(
|
||||
result_set.enabled and result_set.mode != "none"
|
||||
for result_set in config.result_sets
|
||||
)
|
||||
if not config.enabled or (config.mode == "none" and not has_enabled_result_set):
|
||||
continue
|
||||
normalized_displays[skill_name] = config
|
||||
|
||||
@@ -257,13 +265,30 @@ def adapt_reference_batches_with_displays(
|
||||
raw = source.get("raw")
|
||||
if display is not None and isinstance(raw, dict):
|
||||
index_value = int(source.get("index") or len(sources) + 1)
|
||||
display_for_source: SkillDisplayConfig | SkillDisplayResultSetConfig = display
|
||||
result_set_key = source.get("resultSetKey")
|
||||
if result_set_key:
|
||||
display_for_source = next(
|
||||
(
|
||||
result_set
|
||||
for result_set in display.result_sets
|
||||
if result_set.key == result_set_key
|
||||
),
|
||||
display,
|
||||
)
|
||||
elif display.result_sets and display.mode == "none":
|
||||
sources.append(source)
|
||||
continue
|
||||
configured = _normalize_with_display_config(
|
||||
raw,
|
||||
index_value,
|
||||
skill_name,
|
||||
display,
|
||||
display_for_source,
|
||||
)
|
||||
if configured is not None:
|
||||
if isinstance(display_for_source, SkillDisplayResultSetConfig):
|
||||
configured["resultSetKey"] = display_for_source.key
|
||||
configured["resultSetLabel"] = display_for_source.label
|
||||
sources.append(
|
||||
{
|
||||
**source,
|
||||
@@ -318,6 +343,8 @@ def _source_debug_item(source: dict[str, Any]) -> dict[str, Any]:
|
||||
"id": source.get("id"),
|
||||
"skillName": source.get("skillName"),
|
||||
"mode": source.get("mode"),
|
||||
"resultSetKey": source.get("resultSetKey"),
|
||||
"resultSetLabel": source.get("resultSetLabel"),
|
||||
"title": source.get("title"),
|
||||
"source": source.get("source"),
|
||||
"author": source.get("author"),
|
||||
@@ -359,6 +386,16 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
tool_contexts_by_id,
|
||||
)
|
||||
payload = _extract_tool_payload(message)
|
||||
configured_sources = _collect_configured_sources(
|
||||
payload,
|
||||
len(sources) + 1,
|
||||
tool_context,
|
||||
)
|
||||
if configured_sources is not None:
|
||||
for normalized in configured_sources:
|
||||
normalized["index"] = len(sources) + 1
|
||||
sources.append(normalized)
|
||||
continue
|
||||
for raw_item in _first_list(payload):
|
||||
normalized = _normalize_reference_item(
|
||||
raw_item,
|
||||
@@ -373,6 +410,44 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
return sources
|
||||
|
||||
|
||||
def _collect_configured_sources(
|
||||
payload: Any,
|
||||
fallback_index: int,
|
||||
tool_context: dict[str, Any] | None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
display_match = _resolve_display_match({}, payload, tool_context)
|
||||
if display_match is None:
|
||||
return None
|
||||
skill_name, display = display_match
|
||||
configs: list[SkillDisplayConfig | SkillDisplayResultSetConfig] = [
|
||||
result_set
|
||||
for result_set in display.result_sets
|
||||
if result_set.enabled and result_set.mode != "none"
|
||||
]
|
||||
if not configs:
|
||||
configs = [display]
|
||||
|
||||
sources: list[dict[str, Any]] = []
|
||||
for config in configs:
|
||||
raw_items = _list_at_path(payload, config.result_path)
|
||||
if raw_items is None:
|
||||
continue
|
||||
for raw_item in raw_items:
|
||||
normalized = _normalize_with_display_config(
|
||||
raw_item,
|
||||
fallback_index + len(sources),
|
||||
skill_name,
|
||||
config,
|
||||
)
|
||||
if normalized is None:
|
||||
continue
|
||||
if isinstance(config, SkillDisplayResultSetConfig):
|
||||
normalized["resultSetKey"] = config.key
|
||||
normalized["resultSetLabel"] = config.label
|
||||
sources.append(normalized)
|
||||
return sources
|
||||
|
||||
|
||||
def _find_reference_batch_anchor(messages: list[Any], start: int, end: int) -> Any | None:
|
||||
answer = _find_last_assistant_answer(messages, start, end)
|
||||
if answer is not None:
|
||||
@@ -622,6 +697,19 @@ def _first_list(payload: Any) -> list[Any]:
|
||||
return []
|
||||
|
||||
|
||||
def _list_at_path(payload: Any, path: str | None) -> list[dict[str, Any]] | None:
|
||||
cursor = payload
|
||||
if path:
|
||||
for part in path.split("."):
|
||||
if isinstance(cursor, dict):
|
||||
cursor = cursor.get(part)
|
||||
else:
|
||||
return None
|
||||
if not isinstance(cursor, list):
|
||||
return None
|
||||
return [item for item in cursor if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _nested_records(raw: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(raw, dict):
|
||||
return []
|
||||
@@ -654,7 +742,13 @@ def _enabled_skill_displays() -> dict[str, SkillDisplayConfig]:
|
||||
if state.display is not None
|
||||
and state.enabled
|
||||
and state.display.enabled
|
||||
and state.display.mode != "none"
|
||||
and (
|
||||
state.display.mode != "none"
|
||||
or any(
|
||||
result_set.enabled and result_set.mode != "none"
|
||||
for result_set in state.display.result_sets
|
||||
)
|
||||
)
|
||||
}
|
||||
except Exception:
|
||||
return {}
|
||||
@@ -705,7 +799,7 @@ def _normalize_with_display_config(
|
||||
raw: dict[str, Any],
|
||||
fallback_index: int,
|
||||
skill_name: str,
|
||||
display: SkillDisplayConfig,
|
||||
display: SkillDisplayConfig | SkillDisplayResultSetConfig,
|
||||
) -> dict[str, Any] | None:
|
||||
title = _stringify(_get_by_path(raw, display.title_field), f"结果 {fallback_index}")
|
||||
snippet = _stringify(_get_by_path(raw, display.summary_field), "")
|
||||
@@ -737,7 +831,10 @@ def _normalize_with_display_config(
|
||||
}
|
||||
|
||||
|
||||
def _resolve_display_link(raw: dict[str, Any], display: SkillDisplayConfig) -> str | None:
|
||||
def _resolve_display_link(
|
||||
raw: dict[str, Any],
|
||||
display: SkillDisplayConfig | SkillDisplayResultSetConfig,
|
||||
) -> str | None:
|
||||
url = _stringify(_get_by_path(raw, display.url_field), "")
|
||||
item_id = _stringify(_get_by_path(raw, display.id_field), "")
|
||||
if display.link_strategy == "url":
|
||||
|
||||
+9
-1
@@ -1,8 +1,16 @@
|
||||
from .registry import get_scheduled_task_service, set_scheduled_task_service
|
||||
from .service import DuplicateTaskNameError, MarkdownRequirementError, ScheduledTaskService
|
||||
from .service import (
|
||||
DuplicateFavoriteError,
|
||||
DuplicateTaskNameError,
|
||||
HtmlGenerationError,
|
||||
MarkdownRequirementError,
|
||||
ScheduledTaskService,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DuplicateFavoriteError",
|
||||
"DuplicateTaskNameError",
|
||||
"HtmlGenerationError",
|
||||
"MarkdownRequirementError",
|
||||
"ScheduledTaskService",
|
||||
"get_scheduled_task_service",
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class ElementDeliveryClient:
|
||||
self.access_token = access_token
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "ElementDeliveryClient | None":
|
||||
def from_env(cls) -> ElementDeliveryClient | None:
|
||||
enabled = os.environ.get("ELEMENT_ENABLED", "true").strip().lower()
|
||||
if enabled in {"0", "false", "no", "off"}:
|
||||
return None
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
"""HTML page generation helpers for ``html_page`` scheduled tasks.
|
||||
|
||||
Pure, dependency-light utilities the scheduler uses to turn a GLM5 response
|
||||
into a safe, previewable HTML document:
|
||||
|
||||
- :func:`extract_json_object` — fault-tolerant JSON extraction from a model
|
||||
reply that may be wrapped in prose or ```json fences.
|
||||
- :func:`sanitize_html` — strip script-execution vectors (``<script>``,
|
||||
``<iframe>``, ``on*`` handlers, ``javascript:`` URLs, …) using BeautifulSoup.
|
||||
- :func:`validate_html` — cheap structural checks (doctype/html/head/body,
|
||||
non-empty body, no residual scripts) that gate the optional repair pass.
|
||||
- ``build_*_prompt`` — the GLM5 prompt templates (generation / repair / style
|
||||
extraction).
|
||||
- :func:`screenshot_html` — best-effort Playwright screenshot, a no-op when
|
||||
Playwright is not installed (the offline image ships without a browser).
|
||||
|
||||
Kept free of any ``app.*`` import so it stays inside the harness boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tags removed wholesale — they either execute code or load remote active
|
||||
# content that the iframe sandbox alone shouldn't be relied on to contain.
|
||||
_DANGEROUS_TAGS = frozenset(
|
||||
{"script", "iframe", "object", "embed", "applet", "frame", "frameset", "base"}
|
||||
)
|
||||
|
||||
# URL schemes that can execute script when navigated/loaded.
|
||||
_DANGEROUS_URL_RE = re.compile(r"^\s*(javascript|vbscript|data:text/html)", re.IGNORECASE)
|
||||
|
||||
# Attributes carrying a URL we must scheme-check.
|
||||
_URL_ATTRS = frozenset({"href", "src", "action", "formaction", "xlink:href", "background", "poster"})
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
|
||||
"""Split a ``---`` YAML frontmatter block from the body of a SKILL.md.
|
||||
|
||||
Returns ``(frontmatter_dict, body)``. When there is no frontmatter the dict
|
||||
is empty and the body is the whole text.
|
||||
"""
|
||||
if not text or not text.lstrip().startswith("---"):
|
||||
return {}, text or ""
|
||||
lines = text.splitlines()
|
||||
# Skip leading blank lines before the opening fence.
|
||||
start = 0
|
||||
while start < len(lines) and not lines[start].strip():
|
||||
start += 1
|
||||
end = None
|
||||
for i in range(start + 1, len(lines)):
|
||||
if lines[i].strip() == "---":
|
||||
end = i
|
||||
break
|
||||
if end is None:
|
||||
return {}, text
|
||||
fm_block = "\n".join(lines[start + 1 : end])
|
||||
body = "\n".join(lines[end + 1 :])
|
||||
try:
|
||||
import yaml
|
||||
|
||||
data = yaml.safe_load(fm_block) or {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
except Exception:
|
||||
data = {}
|
||||
return data, body
|
||||
|
||||
|
||||
def extract_json_object(text: str | None) -> dict[str, Any] | None:
|
||||
"""Best-effort parse of a JSON object from a model reply.
|
||||
|
||||
Handles three common shapes: a bare JSON object, a ```json fenced block,
|
||||
and a JSON object embedded in surrounding prose. Returns ``None`` when no
|
||||
balanced object can be recovered.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return None
|
||||
candidate = text.strip()
|
||||
|
||||
# Strip a single ```json ... ``` (or ``` ... ```) fence if present.
|
||||
fence = re.search(r"```(?:json)?\s*(.*?)```", candidate, flags=re.DOTALL | re.IGNORECASE)
|
||||
if fence:
|
||||
candidate = fence.group(1).strip()
|
||||
|
||||
# Fast path: the whole thing is already valid JSON.
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Fallback: scan for the first balanced ``{ ... }`` and parse that.
|
||||
start = candidate.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for index in range(start, len(candidate)):
|
||||
char = candidate[index]
|
||||
if in_string:
|
||||
if escape:
|
||||
escape = False
|
||||
elif char == "\\":
|
||||
escape = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
blob = candidate[start : index + 1]
|
||||
try:
|
||||
parsed = json.loads(blob)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
return None
|
||||
|
||||
|
||||
def recover_html_document(text: str | None) -> str | None:
|
||||
"""Slice a complete HTML document out of a model reply.
|
||||
|
||||
Used as a fallback when the JSON wrapper around the ``html`` field can't be
|
||||
parsed — the model commonly emits *literal* newlines and *unescaped* quotes
|
||||
(``"zh-CN"``) inside the value, which is invalid JSON, so
|
||||
:func:`extract_json_object` returns ``None``. Without this, the caller would
|
||||
archive the whole ``{"title": ..., "html": "<!doctype html>…"}`` blob as the
|
||||
page. We locate the ``<!doctype html>`` / ``<html …>`` start and the final
|
||||
``</html>`` and return just that span, dropping the surrounding JSON
|
||||
scaffolding. Also accepts a reply that is already bare HTML. Returns
|
||||
``None`` when no HTML document can be found.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return None
|
||||
lowered = text.lower()
|
||||
start = lowered.find("<!doctype html")
|
||||
if start == -1:
|
||||
start = lowered.find("<html")
|
||||
if start == -1:
|
||||
return None
|
||||
end = lowered.rfind("</html>")
|
||||
candidate = text[start:] if end == -1 else text[start : end + len("</html>")]
|
||||
candidate = candidate.strip()
|
||||
# If the html had been a properly-escaped JSON string value (the JSON failed
|
||||
# to parse for some *other* reason), unescape the common sequences so they
|
||||
# don't survive into the rendered page. Only triggers when escapes exist, so
|
||||
# genuinely-literal HTML (the usual broken-JSON case) is left untouched.
|
||||
if "\\n" in candidate or '\\"' in candidate or "\\/" in candidate or "\\t" in candidate:
|
||||
candidate = (
|
||||
candidate.replace('\\"', '"')
|
||||
.replace("\\n", "\n")
|
||||
.replace("\\r", "\r")
|
||||
.replace("\\t", "\t")
|
||||
.replace("\\/", "/")
|
||||
)
|
||||
return candidate or None
|
||||
|
||||
|
||||
def sanitize_html(html: str) -> tuple[str, list[str]]:
|
||||
"""Remove script-execution vectors from ``html``.
|
||||
|
||||
Returns ``(cleaned_html, removed)`` where ``removed`` is a human-readable
|
||||
report of what was stripped (used for logging / repair feedback). The
|
||||
cleaner is defensive but not a full HTML validator — pair it with an
|
||||
``iframe sandbox`` (no ``allow-scripts``) on the preview side.
|
||||
"""
|
||||
if not html or not isinstance(html, str):
|
||||
return "", []
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except Exception: # pragma: no cover - bs4 is a declared dependency
|
||||
logger.warning("BeautifulSoup unavailable; returning HTML unsanitized")
|
||||
return html, []
|
||||
|
||||
removed: list[str] = []
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# 1) Drop dangerous tags entirely.
|
||||
for tag in soup.find_all(True):
|
||||
name = (tag.name or "").lower()
|
||||
if name in _DANGEROUS_TAGS:
|
||||
removed.append(f"<{name}>")
|
||||
tag.decompose()
|
||||
continue
|
||||
# External stylesheet / preload links pull remote resources.
|
||||
if name == "link":
|
||||
href = str(tag.get("href") or "")
|
||||
rel = " ".join(tag.get("rel") or []) if isinstance(tag.get("rel"), list) else str(tag.get("rel") or "")
|
||||
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
|
||||
removed.append("<link stylesheet>")
|
||||
tag.decompose()
|
||||
continue
|
||||
# meta refresh can redirect to arbitrary URLs.
|
||||
if name == "meta" and str(tag.get("http-equiv") or "").lower() == "refresh":
|
||||
removed.append("<meta refresh>")
|
||||
tag.decompose()
|
||||
continue
|
||||
|
||||
# 2) Strip event handlers and dangerous URL schemes from surviving tags.
|
||||
for tag in soup.find_all(True):
|
||||
for attr in list(tag.attrs.keys()):
|
||||
lowered = attr.lower()
|
||||
if lowered.startswith("on"):
|
||||
removed.append(f"{lowered}=")
|
||||
del tag.attrs[attr]
|
||||
continue
|
||||
if lowered in _URL_ATTRS:
|
||||
value = tag.attrs[attr]
|
||||
value_str = value if isinstance(value, str) else " ".join(value)
|
||||
if _DANGEROUS_URL_RE.match(value_str or ""):
|
||||
removed.append(f"{lowered}={value_str[:24]}")
|
||||
del tag.attrs[attr]
|
||||
|
||||
return str(soup), removed
|
||||
|
||||
|
||||
def validate_html(html: str) -> list[str]:
|
||||
"""Return a list of structural problems with ``html`` (empty == valid).
|
||||
|
||||
A page only counts as a complete document when it has the full skeleton
|
||||
(doctype/html/head/body), a non-empty ``<title>``, visible body content
|
||||
(no white-screen), no residual scripts, and no external stylesheet links.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
if not html or not isinstance(html, str):
|
||||
return ["empty html"]
|
||||
lowered = html.lower()
|
||||
if "<!doctype html" not in lowered:
|
||||
errors.append("缺少 <!doctype html> 声明")
|
||||
if "<html" not in lowered:
|
||||
errors.append("缺少 <html> 根元素")
|
||||
if "<head" not in lowered:
|
||||
errors.append("缺少 <head> 元素")
|
||||
if "<body" not in lowered:
|
||||
errors.append("缺少 <body> 元素")
|
||||
if "<script" in lowered:
|
||||
errors.append("仍包含 <script> 标签(不允许脚本)")
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
# Title must exist and be non-empty.
|
||||
title_el = soup.find("title")
|
||||
if title_el is None or not title_el.get_text(strip=True):
|
||||
errors.append("缺少非空 <title> 标题")
|
||||
# External stylesheet links pull remote resources — not allowed offline.
|
||||
for link in soup.find_all("link"):
|
||||
href = str(link.get("href") or "")
|
||||
rel = " ".join(link.get("rel") or []) if isinstance(link.get("rel"), list) else str(link.get("rel") or "")
|
||||
if re.match(r"^\s*(https?:)?//", href) and "stylesheet" in rel.lower():
|
||||
errors.append("包含外链样式表(不允许外部 CDN/资源)")
|
||||
break
|
||||
# Body must carry some visible content — guards against white-screen.
|
||||
body = soup.body or soup
|
||||
text = body.get_text(strip=True) if body else ""
|
||||
has_media = bool(body.find(["img", "svg", "canvas", "table", "ul", "ol"])) if body else False
|
||||
if len(text) < 5 and not has_media:
|
||||
errors.append("页面正文为空,可能白屏")
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt templates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_GENERATION_TEMPLATE = """你是一个内网页面生成助手。请根据输入数据生成一个完整的 HTML 页面。
|
||||
|
||||
用户页面要求:
|
||||
{page_prompt}
|
||||
{constraint_block}本次数据:
|
||||
{data}
|
||||
|
||||
参考页面风格:
|
||||
{reference_style}
|
||||
|
||||
参考页面布局:
|
||||
{reference_layout}
|
||||
{skill_instructions}
|
||||
要求:
|
||||
1. 只输出 JSON,不要输出 Markdown,不要输出多余解释。
|
||||
2. JSON 字段必须包含 title、html、style_summary、layout_summary。
|
||||
3. html 字段的值必须是合法 JSON 字符串:其中所有换行必须写成 \\n、所有双引号必须转义为 \\",不要在字符串里直接换行或使用未转义的引号。
|
||||
4. html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
|
||||
5. 所有 CSS 写在 <style> 标签内,不要引用外部 CDN 或外链样式表。
|
||||
6. 不要包含 <script>、外链脚本、iframe、表单提交、on* 事件属性。
|
||||
7. 参考页面只用于风格和布局,页面内容必须来自“本次数据”,不要复用参考页旧数据。
|
||||
8. 页面使用中文,排版适合在浏览器中直接预览。
|
||||
"""
|
||||
|
||||
_REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问题,请修复后重新输出完整页面。
|
||||
|
||||
原始页面要求:
|
||||
{requirements}
|
||||
|
||||
发现的问题:
|
||||
{errors}
|
||||
|
||||
待修复 HTML:
|
||||
{html}
|
||||
|
||||
要求:
|
||||
1. 只输出 JSON,字段包含 fixed_html、changes(修复说明数组)。
|
||||
2. fixed_html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
|
||||
3. 移除所有 <script>、iframe、外链脚本、on* 事件属性、表单提交。
|
||||
4. 保留原有内容与数据,只修复结构与安全问题。
|
||||
"""
|
||||
|
||||
_STYLE_EXTRACT_TEMPLATE = """你是一个页面风格分析助手。请分析下面的 HTML 页面,提取可复用的风格与布局特征。
|
||||
|
||||
HTML 页面:
|
||||
{html}
|
||||
|
||||
要求:
|
||||
1. 只输出 JSON,字段包含 style_summary、layout_summary、content_structure_summary、reuse_prompt。
|
||||
2. style_summary:颜色、字体、间距、圆角、阴影、视觉密度等视觉风格。
|
||||
3. layout_summary:页面分区、卡片、表格、图表、导航等布局结构。
|
||||
4. content_structure_summary:信息组织顺序与层级。
|
||||
5. reuse_prompt:一段可直接拼接到下次生成 prompt 的中文风格说明,不要包含本页的具体数据。
|
||||
"""
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int) -> str:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||||
text = text or ""
|
||||
return text if len(text) <= limit else text[:limit] + "\n…(内容过长已截断)"
|
||||
|
||||
|
||||
def build_generation_prompt(
|
||||
*,
|
||||
page_prompt: str,
|
||||
negative_prompt: str | None,
|
||||
data: Any,
|
||||
reference_style: str | None,
|
||||
reference_layout: str | None,
|
||||
skill_instructions: str | None = None,
|
||||
data_limit: int = 16000,
|
||||
skill_limit: int = 30000,
|
||||
) -> str:
|
||||
skill_block = ""
|
||||
if skill_instructions and skill_instructions.strip():
|
||||
skill_block = "\n可用页面生成技能规范(请严格遵循):\n" + _clip(skill_instructions.strip(), skill_limit) + "\n"
|
||||
# The constraint block is only emitted when the user actually supplied one,
|
||||
# and is framed as a top-priority hard rule that overrides the page
|
||||
# requirements / reference page / data on conflict — otherwise the model
|
||||
# tends to treat "限制条件" as a soft hint and ignore it.
|
||||
constraint_block = ""
|
||||
neg = (negative_prompt or "").strip()
|
||||
if neg:
|
||||
constraint_block = (
|
||||
"\n限制条件(最高优先级的硬性约束,必须逐条严格遵守;"
|
||||
"若与上面的页面要求、参考页面或本次数据发生冲突,一律以这里的限制条件为准):\n"
|
||||
+ neg
|
||||
+ "\n\n"
|
||||
)
|
||||
return _GENERATION_TEMPLATE.format(
|
||||
page_prompt=(page_prompt or "(未提供,使用通用看板风格)").strip(),
|
||||
constraint_block=constraint_block,
|
||||
data=_clip(data, data_limit),
|
||||
reference_style=(reference_style or "无").strip(),
|
||||
reference_layout=(reference_layout or "无").strip(),
|
||||
skill_instructions=skill_block,
|
||||
)
|
||||
|
||||
|
||||
def build_repair_prompt(*, html: str, errors: list[str], requirements: str | None) -> str:
|
||||
return _REPAIR_TEMPLATE.format(
|
||||
requirements=(requirements or "无").strip(),
|
||||
errors="\n".join(f"- {e}" for e in errors) or "- 未知问题",
|
||||
html=_clip(html, 32000),
|
||||
)
|
||||
|
||||
|
||||
def build_style_extract_prompt(*, html: str) -> str:
|
||||
return _STYLE_EXTRACT_TEMPLATE.format(html=_clip(html, 32000))
|
||||
|
||||
|
||||
async def screenshot_html(html: str, output_path) -> str | None:
|
||||
"""Best-effort PNG screenshot of ``html`` for white-screen / console QA.
|
||||
|
||||
Returns the screenshot path on success, or ``None`` when Playwright (or a
|
||||
usable browser) is unavailable — the offline image ships without one, so
|
||||
this must degrade silently rather than fail the run.
|
||||
"""
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch()
|
||||
try:
|
||||
page = await browser.new_page()
|
||||
await page.set_content(html, wait_until="networkidle")
|
||||
await page.screenshot(path=str(output_path), full_page=True)
|
||||
finally:
|
||||
await browser.close()
|
||||
return str(output_path)
|
||||
except Exception:
|
||||
logger.debug("Playwright screenshot failed; skipping QA screenshot", exc_info=True)
|
||||
return None
|
||||
+488
-1
@@ -22,6 +22,7 @@ from deerflow.config.agents_config import AGENT_ID_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.scheduler import html_page as html_page_utils
|
||||
from deerflow.runtime.scheduler.element import ElementDeliveryClient
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.utils.time import now_iso
|
||||
@@ -36,6 +37,11 @@ _RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
|
||||
# Retries allowed after the first attempt before the run is marked "failed".
|
||||
_MAX_RUN_RETRIES = 3
|
||||
|
||||
# Max HTML length (chars) inlined into the run's ``result_json``. Beyond this
|
||||
# the inline copy is dropped to stay under MySQL's 64 KB ``TEXT`` cap; the full
|
||||
# document is always retrievable from the page store (LONGTEXT) + index.html.
|
||||
_MAX_INLINE_RESULT_HTML = 60000
|
||||
|
||||
# Minimum length (in characters) the generated Markdown file's content must
|
||||
# reach when a task has ``require_markdown`` enabled. The model is NOT told about
|
||||
# this threshold — it is enforced silently by the service.
|
||||
@@ -58,6 +64,22 @@ class MarkdownRequirementError(Exception):
|
||||
"""
|
||||
|
||||
|
||||
class HtmlGenerationError(Exception):
|
||||
"""Raised when an ``html_page`` run did not produce a complete, valid HTML
|
||||
document (missing doctype/html/head/body, empty body, or residual scripts
|
||||
after the sanitize + repair passes). The run is marked failed — a page only
|
||||
counts as successful when it is a full, previewable HTML document."""
|
||||
|
||||
|
||||
class DuplicateFavoriteError(Exception):
|
||||
"""A user already bookmarked this page. Surfaced as HTTP 409 by the router
|
||||
so the same page cannot be favorited twice."""
|
||||
|
||||
def __init__(self, page_id: str) -> None:
|
||||
self.page_id = page_id
|
||||
super().__init__(f"Page {page_id!r} is already favorited")
|
||||
|
||||
|
||||
class DuplicateTaskNameError(Exception):
|
||||
"""A user already owns a scheduled task with the requested name.
|
||||
|
||||
@@ -835,7 +857,7 @@ class ScheduledTaskService:
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
|
||||
self._run_task_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
|
||||
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except TimeoutError:
|
||||
@@ -875,6 +897,342 @@ class ScheduledTaskService:
|
||||
await self._deliver_element_result(task, result["content"])
|
||||
return
|
||||
|
||||
async def _run_task_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Dispatch a single execution attempt by task kind.
|
||||
|
||||
``html_page`` tasks render an HTML document via GLM5; everything else
|
||||
runs the regular lead-agent attempt. The kind lives in
|
||||
``execution_context.task_kind`` so no schema migration is needed.
|
||||
"""
|
||||
ctx = task.get("execution_context")
|
||||
kind = ctx.get("task_kind") if isinstance(ctx, dict) else None
|
||||
if kind == "html_page":
|
||||
return await self._run_html_page_attempt(
|
||||
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
|
||||
)
|
||||
return await self._run_agent_attempt(
|
||||
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _html_config(task: dict[str, Any]) -> dict[str, Any]:
|
||||
ctx = task.get("execution_context")
|
||||
cfg = ctx.get("html_config") if isinstance(ctx, dict) else None
|
||||
return cfg if isinstance(cfg, dict) else {}
|
||||
|
||||
async def _invoke_chat_model(self, model_name: str | None, prompt: str) -> str:
|
||||
"""Invoke a chat model once and return its text content.
|
||||
|
||||
Used for the GLM5 HTML generation / repair / style-extraction passes.
|
||||
Thinking is disabled to keep structured-JSON output stable.
|
||||
"""
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
# No hardcoded model: use exactly the selected model; only when none was
|
||||
# selected do we fall back to the system default (config models[0]).
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
response = await model.ainvoke([HumanMessage(content=prompt)])
|
||||
content = getattr(response, "content", response)
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [part.get("text", "") if isinstance(part, dict) else str(part) for part in content]
|
||||
return "\n".join(p for p in parts if p)
|
||||
return str(content)
|
||||
|
||||
async def _gather_html_data(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> str:
|
||||
"""Run the lead agent to collect source data for the HTML page.
|
||||
|
||||
Reuses the regular agent-run primitives (with the task's search query
|
||||
folded into the prompt) and returns the agent's final text, which feeds
|
||||
the GLM5 render step as ``本次数据``. Returns an empty string if the
|
||||
agent produced nothing.
|
||||
"""
|
||||
cfg = self._html_config(task)
|
||||
page_prompt = str(cfg.get("page_prompt") or "").strip()
|
||||
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
|
||||
gather_instruction = (
|
||||
"请运用你自身的检索/搜索等技能,围绕下述页面需求获取最新、真实的资料,"
|
||||
"并整理成结构化的中文要点(可用列表/小标题),供后续生成 HTML 页面使用,本步骤不要生成 HTML:\n"
|
||||
)
|
||||
parts = [gather_instruction]
|
||||
if page_prompt:
|
||||
parts.append(f"页面需求:{page_prompt}")
|
||||
base_prompt = str(task.get("prompt") or "").strip()
|
||||
if base_prompt:
|
||||
parts.append(f"补充说明:{base_prompt}")
|
||||
# Apply the user's constraints to data gathering too, not just rendering
|
||||
# — a "限制条件" like "只要最近 24 小时的新闻" or "排除某主题" must shape
|
||||
# which sources/data the agent collects, otherwise it silently leaks in.
|
||||
if negative_prompt:
|
||||
parts.append(f"限制条件(最高优先级,必须严格遵守):{negative_prompt}")
|
||||
gather_task = {**task, "prompt": "\n".join(parts)}
|
||||
# Never apply the markdown gate to the gather pass.
|
||||
gather_ctx = dict(task.get("execution_context") or {})
|
||||
gather_ctx.pop("require_markdown", None)
|
||||
gather_ctx.pop("task_kind", None)
|
||||
gather_ctx.pop("html_config", None)
|
||||
# Use the task's selected model for the data-gathering agent too, so the
|
||||
# whole HTML pipeline honors ``html_config.model_name`` instead of
|
||||
# silently falling back to the default model for this step.
|
||||
gather_model = str(cfg.get("model_name") or "").strip()
|
||||
if gather_model:
|
||||
gather_ctx["model_name"] = gather_model
|
||||
gather_task["execution_context"] = gather_ctx
|
||||
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
record = await self._start_agent_run(gather_task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
return text or ""
|
||||
|
||||
def _collect_html_skill_instructions(self) -> str:
|
||||
"""Concatenate the bodies of enabled skills marked for HTML pages.
|
||||
|
||||
A skill opts in via ``metadata.kind: html_page`` in its SKILL.md
|
||||
frontmatter. Only enabled skills (per the skills settings the user
|
||||
toggles) are injected, so the render step honors the user's skill
|
||||
selection. Best-effort — any failure yields an empty string.
|
||||
"""
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
except Exception:
|
||||
logger.debug("Failed to load skills for HTML render injection", exc_info=True)
|
||||
return ""
|
||||
blocks: list[str] = []
|
||||
for skill in skills:
|
||||
try:
|
||||
text = Path(skill.skill_file).read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
continue
|
||||
frontmatter, body = html_page_utils.parse_frontmatter(text)
|
||||
meta = frontmatter.get("metadata") if isinstance(frontmatter, dict) else None
|
||||
if not (isinstance(meta, dict) and str(meta.get("kind")) == "html_page"):
|
||||
continue
|
||||
body = body.strip()
|
||||
if body:
|
||||
blocks.append(f"### 技能:{skill.name}\n{body}")
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
def _archive_html_index(self, run_id: str, html: str) -> Path | None:
|
||||
"""Write the generated HTML to the run's archive dir as ``index.html``."""
|
||||
try:
|
||||
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id / "files"
|
||||
archive_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = archive_dir / "index.html"
|
||||
target.write_text(html, encoding="utf-8")
|
||||
return target
|
||||
except Exception:
|
||||
logger.warning("Failed to archive generated HTML for run %s", run_id, exc_info=True)
|
||||
return None
|
||||
|
||||
async def _run_html_page_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate an HTML page for an ``html_page`` task.
|
||||
|
||||
Pipeline: gather data via the lead agent → optionally load a reference
|
||||
favorite's style/layout → GLM5 render → sanitize → validate → (one
|
||||
GLM5 repair pass if invalid) → best-effort screenshot → archive
|
||||
``index.html`` → persist a ``scheduled_html_pages`` row → build the run
|
||||
result payload (``format: "html"`` with the full HTML inline).
|
||||
"""
|
||||
cfg = self._html_config(task)
|
||||
# The user's selected model only — no hardcoded GLM-5. Empty → None →
|
||||
# system default (config models[0]).
|
||||
model_name = str(cfg.get("model_name") or "").strip() or None
|
||||
page_prompt = str(cfg.get("page_prompt") or "").strip()
|
||||
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
|
||||
reference_favorite_id = cfg.get("reference_favorite_id") or None
|
||||
|
||||
# 1) Gather source data (real, via the agent's search tools).
|
||||
data_text = await self._gather_html_data(
|
||||
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
|
||||
)
|
||||
|
||||
# 2) Load reference style/layout from a favorite if one was selected.
|
||||
reference_style = None
|
||||
reference_layout = None
|
||||
if reference_favorite_id:
|
||||
store = self._html_store
|
||||
if store is not None:
|
||||
try:
|
||||
favorite = await store.get_favorite(str(reference_favorite_id))
|
||||
except Exception:
|
||||
favorite = None
|
||||
if favorite is not None:
|
||||
reference_style = favorite.get("reuse_prompt") or favorite.get("style_summary")
|
||||
reference_layout = favorite.get("layout_summary")
|
||||
|
||||
# 3) Render with the chosen model, injecting any enabled HTML skills.
|
||||
skill_instructions = self._collect_html_skill_instructions()
|
||||
gen_prompt = html_page_utils.build_generation_prompt(
|
||||
page_prompt=page_prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
data=data_text,
|
||||
reference_style=reference_style,
|
||||
reference_layout=reference_layout,
|
||||
skill_instructions=skill_instructions,
|
||||
)
|
||||
raw = await self._invoke_chat_model(model_name, gen_prompt)
|
||||
parsed = html_page_utils.extract_json_object(raw) or {}
|
||||
title = str(parsed.get("title") or task.get("name") or "生成页面").strip()
|
||||
html = parsed.get("html")
|
||||
if not isinstance(html, str) or "<" not in html:
|
||||
# The JSON wrapper couldn't be parsed (the model emitted literal
|
||||
# newlines / unescaped quotes inside the "html" value), or it replied
|
||||
# with bare HTML. Either way, recover just the HTML document so we
|
||||
# never archive the {"title":...,"html":"..."} JSON blob as the page.
|
||||
recovered = html_page_utils.recover_html_document(raw)
|
||||
if recovered is None:
|
||||
raise ValueError("模型未返回有效的 HTML(JSON 解析失败)")
|
||||
html = recovered
|
||||
style_summary = str(parsed.get("style_summary") or "").strip() or None
|
||||
layout_summary = str(parsed.get("layout_summary") or "").strip() or None
|
||||
|
||||
# 4) Sanitize + validate, with one repair pass on failure.
|
||||
html, _removed = html_page_utils.sanitize_html(html)
|
||||
errors = html_page_utils.validate_html(html)
|
||||
if errors:
|
||||
logger.info("Generated HTML failed validation (%s); attempting repair", errors)
|
||||
repair_prompt = html_page_utils.build_repair_prompt(
|
||||
html=html, errors=errors, requirements=page_prompt or task.get("prompt")
|
||||
)
|
||||
try:
|
||||
repaired_raw = await self._invoke_chat_model(model_name, repair_prompt)
|
||||
repaired = html_page_utils.extract_json_object(repaired_raw) or {}
|
||||
fixed = repaired.get("fixed_html")
|
||||
if isinstance(fixed, str) and "<" in fixed:
|
||||
fixed_clean, _ = html_page_utils.sanitize_html(fixed)
|
||||
if not html_page_utils.validate_html(fixed_clean):
|
||||
html = fixed_clean
|
||||
except Exception:
|
||||
logger.warning("HTML repair pass failed; keeping sanitized original", exc_info=True)
|
||||
|
||||
# 4b) Completeness gate — a page only counts as successful when it is a
|
||||
# full, valid HTML document. If it still fails validation after the
|
||||
# repair pass, fail the run (no half-baked page is archived/persisted).
|
||||
final_errors = html_page_utils.validate_html(html)
|
||||
if final_errors:
|
||||
raise HtmlGenerationError(
|
||||
"HTML 生成未通过完整性校验:" + ";".join(final_errors) + "。请调整页面要求或重试。"
|
||||
)
|
||||
|
||||
# 5) Best-effort screenshot QA (no-op without Playwright).
|
||||
screenshot_path = None
|
||||
try:
|
||||
shot_target = get_paths().base_dir / "scheduled-task-runs" / run_id / "files" / "preview.png"
|
||||
shot_target.parent.mkdir(parents=True, exist_ok=True)
|
||||
screenshot_path = await html_page_utils.screenshot_html(html, shot_target)
|
||||
except Exception:
|
||||
screenshot_path = None
|
||||
|
||||
# 6) Archive index.html.
|
||||
archive_path = self._archive_html_index(run_id, html)
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
if archive_path is not None:
|
||||
files.append(
|
||||
{
|
||||
"name": "index.html",
|
||||
"path": str(archive_path),
|
||||
"virtual_path": "/mnt/user-data/outputs/index.html",
|
||||
"mime_type": "text/html",
|
||||
"size": archive_path.stat().st_size if archive_path.exists() else None,
|
||||
"kind": "html",
|
||||
}
|
||||
)
|
||||
if screenshot_path:
|
||||
shot = Path(screenshot_path)
|
||||
files.append(
|
||||
{
|
||||
"name": "preview.png",
|
||||
"path": str(shot),
|
||||
"mime_type": "image/png",
|
||||
"size": shot.stat().st_size if shot.exists() else None,
|
||||
"kind": "artifact",
|
||||
}
|
||||
)
|
||||
|
||||
data_snapshot = {"data_text": data_text[:8000] if isinstance(data_text, str) else ""}
|
||||
prompt_snapshot = {
|
||||
"page_prompt": page_prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"reference_favorite_id": reference_favorite_id,
|
||||
"model_name": model_name,
|
||||
}
|
||||
|
||||
# 7) Persist the page asset (best-effort — failure must not fail the run).
|
||||
store = self._html_store
|
||||
if store is not None:
|
||||
try:
|
||||
await store.create_page(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"run_id": run_id,
|
||||
"user_id": task["user_id"],
|
||||
"title": title,
|
||||
"html_content": html,
|
||||
"html_file_path": str(archive_path) if archive_path else None,
|
||||
"data_snapshot": data_snapshot,
|
||||
"prompt_snapshot": prompt_snapshot,
|
||||
"reference_favorite_id": reference_favorite_id,
|
||||
"style_summary": style_summary,
|
||||
"layout_summary": layout_summary,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist scheduled_html_pages row for run %s", run_id, exc_info=True)
|
||||
|
||||
task_tz = ZoneInfo(task.get("timezone") or "UTC")
|
||||
local_time = scheduled_for.astimezone(task_tz)
|
||||
# ``scheduled_task_runs.result_json`` is TEXT (64 KB) on MySQL, so only
|
||||
# inline the HTML when it comfortably fits. The full document always
|
||||
# lives in ``scheduled_html_pages.html_content`` (LONGTEXT) + the
|
||||
# archived ``index.html``; the ``/runs/{id}/html`` endpoints read from
|
||||
# there, so previews work even when the inline copy is omitted here.
|
||||
inline_html = html if len(html) <= _MAX_INLINE_RESULT_HTML else None
|
||||
return {
|
||||
"title": f"{title} - {local_time.strftime('%Y-%m-%d %H:%M')}",
|
||||
"content": page_prompt or f"HTML 页面已生成:{title}",
|
||||
"format": "html",
|
||||
"html": inline_html,
|
||||
"style_summary": style_summary,
|
||||
"layout_summary": layout_summary,
|
||||
"data_snapshot": data_snapshot,
|
||||
"prompt_snapshot": prompt_snapshot,
|
||||
"files": files,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
async def _run_agent_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
@@ -1275,3 +1633,132 @@ class ScheduledTaskService:
|
||||
|
||||
async def unsubscribe_task(self, user_id: str, task_id: str) -> bool:
|
||||
return await self.store.delete_subscription(task_id, user_id)
|
||||
|
||||
# --- HTML pages / favorites ---
|
||||
|
||||
@property
|
||||
def _html_store(self):
|
||||
state = getattr(self.app, "state", None)
|
||||
return getattr(state, "html_page_store", None)
|
||||
|
||||
async def get_html_page_for_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Return the generated HTML page for a run (page-store row or run result)."""
|
||||
store = self._html_store
|
||||
if store is not None:
|
||||
page = await store.get_page_by_run(run_id)
|
||||
if page is not None:
|
||||
return page
|
||||
# Fallback to the inline HTML on the run result.
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
result = run.get("result") if isinstance(run.get("result"), dict) else None
|
||||
if not result or not isinstance(result.get("html"), str):
|
||||
return None
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"task_id": run.get("task_id"),
|
||||
"title": result.get("title"),
|
||||
"html_content": result.get("html"),
|
||||
"style_summary": result.get("style_summary"),
|
||||
"layout_summary": result.get("layout_summary"),
|
||||
"created_at": result.get("created_at"),
|
||||
}
|
||||
|
||||
async def get_latest_html_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
store = self._html_store
|
||||
if store is not None:
|
||||
page = await store.latest_page_for_task(task_id)
|
||||
if page is not None:
|
||||
return page
|
||||
# Fallback: newest run that carries inline HTML.
|
||||
runs = await self.store.list_runs_any(task_id, limit=20)
|
||||
for run in runs:
|
||||
result = run.get("result") if isinstance(run.get("result"), dict) else None
|
||||
if result and isinstance(result.get("html"), str):
|
||||
return await self.get_html_page_for_run(run["id"])
|
||||
return None
|
||||
|
||||
async def list_html_page_favorites(self, user_id: str) -> list[dict[str, Any]]:
|
||||
store = self._html_store
|
||||
if store is None:
|
||||
return []
|
||||
return await store.list_favorites(user_id)
|
||||
|
||||
async def get_html_page_favorite(self, user_id: str, favorite_id: str) -> dict[str, Any] | None:
|
||||
store = self._html_store
|
||||
if store is None:
|
||||
return None
|
||||
favorite = await store.get_favorite(favorite_id)
|
||||
if favorite is None or favorite.get("user_id") != user_id:
|
||||
return None
|
||||
return favorite
|
||||
|
||||
async def delete_html_page_favorite(self, user_id: str, favorite_id: str) -> bool:
|
||||
store = self._html_store
|
||||
if store is None:
|
||||
return False
|
||||
return await store.delete_favorite(favorite_id, user_id)
|
||||
|
||||
async def create_html_page_favorite(
|
||||
self,
|
||||
user_id: str,
|
||||
page_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
description: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Bookmark a generated page, extracting a reusable style summary via GLM5.
|
||||
|
||||
Returns ``None`` when the page does not exist or is not owned by the
|
||||
caller. Style extraction is best-effort — a failure still produces a
|
||||
favorite (without the reuse hints) so the user's bookmark is never lost.
|
||||
"""
|
||||
store = self._html_store
|
||||
if store is None:
|
||||
return None
|
||||
page = await store.get_page(page_id)
|
||||
if page is None or page.get("user_id") != user_id:
|
||||
return None
|
||||
# A page can only be bookmarked once per user.
|
||||
existing = await store.get_favorite_by_page(user_id, page_id)
|
||||
if existing is not None:
|
||||
raise DuplicateFavoriteError(page_id)
|
||||
html = page.get("html_content") or ""
|
||||
|
||||
style_summary = page.get("style_summary")
|
||||
layout_summary = page.get("layout_summary")
|
||||
content_structure_summary = None
|
||||
reuse_prompt = None
|
||||
if html:
|
||||
try:
|
||||
# Reuse the model the page was generated with (recorded in the
|
||||
# page's prompt snapshot) so favoriting doesn't fall back to the
|
||||
# default model.
|
||||
extract_model = (page.get("prompt_snapshot") or {}).get("model_name")
|
||||
extract_prompt = html_page_utils.build_style_extract_prompt(html=html)
|
||||
raw = await self._invoke_chat_model(extract_model, extract_prompt)
|
||||
extracted = html_page_utils.extract_json_object(raw) or {}
|
||||
style_summary = extracted.get("style_summary") or style_summary
|
||||
layout_summary = extracted.get("layout_summary") or layout_summary
|
||||
content_structure_summary = extracted.get("content_structure_summary")
|
||||
reuse_prompt = extracted.get("reuse_prompt")
|
||||
except Exception:
|
||||
logger.warning("HTML style extraction failed for page %s; saving favorite without hints", page_id, exc_info=True)
|
||||
|
||||
return await store.create_favorite(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"user_id": user_id,
|
||||
"page_id": page_id,
|
||||
"title": (title or page.get("title") or "收藏页面").strip(),
|
||||
"description": (description or None),
|
||||
"tags": tags or [],
|
||||
"html_snapshot": html,
|
||||
"style_summary": style_summary,
|
||||
"layout_summary": layout_summary,
|
||||
"content_structure_summary": content_structure_summary,
|
||||
"reuse_prompt": reuse_prompt,
|
||||
}
|
||||
)
|
||||
@@ -6,11 +6,13 @@ from .present_file_tool import present_file_tool
|
||||
from .router_quality_tool import router_quality_check
|
||||
from .scheduled_task_tool import scheduled_task_tool
|
||||
from .setup_agent_tool import setup_agent
|
||||
from .setup_writing_tool import setup_writing
|
||||
from .task_tool import task_tool
|
||||
from .view_image_tool import view_image_tool
|
||||
|
||||
__all__ = [
|
||||
"setup_agent",
|
||||
"setup_writing",
|
||||
"present_file_tool",
|
||||
"ask_clarification_tool",
|
||||
"agent_orchestration_tool",
|
||||
|
||||
+1
-1
@@ -56,7 +56,7 @@ def notebook_search_tool(query: str, top_k: int = 10) -> str:
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"query": query,
|
||||
"source_ids": [source_ids],
|
||||
"source_ids": source_ids,
|
||||
"search_space_id": search_space_id,
|
||||
"top_k": top_k,
|
||||
},
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ def setup_agent(
|
||||
description: One-line description of what the agent does.
|
||||
name: Optional human-readable display name for the agent. If omitted,
|
||||
the agent's stable id is used as the display name.
|
||||
skills: Optional list of skill names this agent should use. None or empty means no skills.
|
||||
skills: Optional list of skill names this agent should use. None or empty means no skills. Maximum 5 skills allowed.
|
||||
"""
|
||||
|
||||
agent_name: str | None = (runtime.context.get("agent_id") or runtime.context.get("agent_name")) if runtime.context else None
|
||||
@@ -48,7 +48,7 @@ def setup_agent(
|
||||
if description:
|
||||
config_data["description"] = description
|
||||
if skills is not None:
|
||||
config_data["skills"] = skills
|
||||
config_data["skills"] = skills[:5]
|
||||
|
||||
config_file = agent_dir / "config.yaml"
|
||||
with open(config_file, "w", encoding="utf-8") as f:
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""写作配置助手用的提交工具。
|
||||
|
||||
与 ``setup_agent`` 同构:模型在「写作配置 Q&A」里与用户聊清需求后,调用本工具把建议的
|
||||
写作配置「提交」出来。实际它总是被 ``SetupWritingApprovalMiddleware`` 在 ``after_model``
|
||||
拦截(不真正执行)——拦截后把建议字段打包进审批 ToolMessage,前端据此回填左侧写作表单。
|
||||
|
||||
因此本工具的函数体基本是兜底:万一未被拦截(如审批被预先放行),也只回一句确认、不落任何盘。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.prebuilt import ToolRuntime
|
||||
from langgraph.types import Command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@tool
|
||||
def setup_writing(
|
||||
user_intent: str,
|
||||
runtime: ToolRuntime,
|
||||
article_type: str | None = None,
|
||||
word_count_target: int | None = None,
|
||||
target_audience: str | None = None,
|
||||
writing_mode: str | None = None,
|
||||
material_source: str | None = None,
|
||||
outline: str | None = None,
|
||||
) -> Command:
|
||||
"""把与用户聊清的写作配置提交给前端表单(不直接开始写作)。
|
||||
|
||||
Args:
|
||||
user_intent: 用户的写作意图(精炼成一句话的写作目标)。
|
||||
article_type: 文章类型的 key(须是系统给定的合法类型之一;不确定可留空)。
|
||||
word_count_target: 目标字数(正整数)。
|
||||
target_audience: 目标读者(如「行业专家」「普通读者」)。
|
||||
writing_mode: 写作模式,'strict'(严格依据素材)或 'loose'(可结合通用知识)。
|
||||
material_source: 素材来源,'knowledge_base'(知识库)或 'skill'(技能检索)。
|
||||
outline: 若用户已提供大纲,原样填入其大纲文本(保留其层级与标题),否则留空。
|
||||
"""
|
||||
logger.info("[setup_writing] proposed writing config submitted (intercepted by approval middleware)")
|
||||
return Command(
|
||||
update={
|
||||
"messages": [
|
||||
ToolMessage(
|
||||
content="写作配置已生成,请在右侧确认后应用到表单。",
|
||||
tool_call_id=runtime.tool_call_id,
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
+37
-5
@@ -51,20 +51,33 @@ def view_image_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
image_path: str,
|
||||
tool_call_id: Annotated[str, InjectedToolCallId],
|
||||
question: str | None = None,
|
||||
) -> Command:
|
||||
"""Read an image file.
|
||||
"""Read and understand an image file.
|
||||
|
||||
Use this tool to read an image file and make it available for display.
|
||||
This is the ONLY correct way to understand an image. The image is analyzed by
|
||||
the configured vision model and its description/answer is returned to you as
|
||||
the tool result. Treat that result as authoritative.
|
||||
|
||||
When to use the view_image tool:
|
||||
- When you need to view an image file.
|
||||
- Whenever you need to view, describe, or answer a question about an image file.
|
||||
|
||||
When NOT to use the view_image tool:
|
||||
- For non-image files (use present_files instead)
|
||||
- For multiple files at once (use present_files instead)
|
||||
|
||||
IMPORTANT — do NOT attempt to understand images any other way:
|
||||
- Never read, open, decode, or parse an image with read_file, grep, bash,
|
||||
python, PIL/Pillow, OpenCV, EXIF parsing, or byte/hex inspection. Those
|
||||
cannot interpret visual content and will only waste turns.
|
||||
- If `view_image` returns a weak or incomplete description, do not fall back
|
||||
to code. Either call `view_image` again with a more specific `question`,
|
||||
or tell the user the configured vision model could not extract more detail
|
||||
(the vision model may need to be changed in config).
|
||||
|
||||
Args:
|
||||
image_path: Absolute /mnt/user-data virtual path to the image file. Common formats supported: jpg, jpeg, png, webp.
|
||||
question: Optional specific question to ask about the image (e.g. "What is the total on this invoice?"). When omitted, a general detailed description is produced.
|
||||
"""
|
||||
from deerflow.sandbox.exceptions import SandboxRuntimeError
|
||||
from deerflow.sandbox.tools import (
|
||||
@@ -153,8 +166,27 @@ def view_image_tool(
|
||||
mime_type = detected_mime_type
|
||||
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
||||
|
||||
# Update viewed_images in state
|
||||
# The merge_viewed_images reducer will handle merging with existing images
|
||||
# Delegated recognition: when a dedicated vision model is configured, send
|
||||
# the image to that model and return its description as the tool result, so
|
||||
# a text-only main chat model can still "see" the image.
|
||||
from deerflow.models.vision import is_delegated_vision, recognize_image
|
||||
|
||||
if is_delegated_vision():
|
||||
try:
|
||||
description = recognize_image(image_base64, mime_type, question=question)
|
||||
except Exception as e: # noqa: BLE001 - surface as a recoverable tool error
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Error recognizing image: {_sanitize_image_error(e, thread_data)}", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
if not description:
|
||||
description = "The vision model returned an empty description."
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(f"Image recognition result for {image_path}:\n\n{description}", tool_call_id=tool_call_id)]},
|
||||
)
|
||||
|
||||
# Native vision: store the image so ViewImageMiddleware injects it into the
|
||||
# (vision-capable) main model before the next LLM call.
|
||||
# The merge_viewed_images reducer will handle merging with existing images.
|
||||
new_viewed_images = {image_path: {"base64": image_base64, "mime_type": mime_type}}
|
||||
|
||||
return Command(
|
||||
|
||||
@@ -142,11 +142,14 @@ def get_available_tools(
|
||||
if model_name is None and config.models:
|
||||
model_name = config.models[0].name
|
||||
|
||||
# Add view_image_tool only if the model supports vision
|
||||
model_config = config.get_model_config(model_name) if model_name else None
|
||||
if model_config is not None and model_config.supports_vision:
|
||||
# Add view_image_tool when vision is available — either the main model
|
||||
# supports vision natively, or a dedicated vision model is configured
|
||||
# (vision.model_name) so recognition can be delegated to it.
|
||||
from deerflow.models.vision import vision_enabled
|
||||
|
||||
if vision_enabled(model_name, app_config=config):
|
||||
builtin_tools.append(view_image_tool)
|
||||
logger.info(f"Including view_image_tool for model '{model_name}' (supports_vision=True)")
|
||||
logger.info(f"Including view_image_tool for model '{model_name}' (vision available)")
|
||||
|
||||
# Get cached MCP tools if enabled
|
||||
# NOTE: We use ExtensionsConfig.from_file() instead of config.extensions
|
||||
|
||||
@@ -138,12 +138,21 @@ def open_upload_file_no_symlink(base_dir: Path, filename: str) -> tuple[Path, ob
|
||||
|
||||
validate_path_traversal(dest, base_dir)
|
||||
|
||||
if not hasattr(os, "O_NOFOLLOW"):
|
||||
raise UnsafeUploadPathError("Upload writes require O_NOFOLLOW support")
|
||||
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW
|
||||
# ``O_NOFOLLOW`` closes the TOCTOU window between the ``lstat`` above and the
|
||||
# ``open`` below, but it is a POSIX-only flag — it does not exist on Windows
|
||||
# (``os.O_NOFOLLOW`` is undefined there). Windows also lacks ``O_NONBLOCK``
|
||||
# and needs ``O_BINARY`` to avoid CRLF translation on the raw fd. Where
|
||||
# ``O_NOFOLLOW`` is unavailable we still reject pre-existing symlinks via the
|
||||
# ``lstat`` check and re-verify the opened file with ``fstat`` below; the
|
||||
# residual race only matters under a hostile local user, which is out of
|
||||
# scope for the single-user local deployment this path targets.
|
||||
flags = os.O_WRONLY | os.O_CREAT
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
if hasattr(os, "O_NONBLOCK"):
|
||||
flags |= os.O_NONBLOCK
|
||||
if hasattr(os, "O_BINARY"):
|
||||
flags |= os.O_BINARY
|
||||
|
||||
try:
|
||||
fd = os.open(dest, flags, 0o600)
|
||||
|
||||
Reference in new issue
Block a user