This commit is contained in:
xuhd committed 2026-05-19 09:40:19 +08:00
commit 070cd07199
372 files changed
+86240 -4969

No files matched your search

@@ -0,0 +1,3 @@
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
@@ -0,0 +1,231 @@
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
from __future__ import annotations
from langgraph.checkpoint.memory import MemorySaver
from langgraph.graph import END, StateGraph
from langgraph.types import interrupt
from .state import AIWritingState
from .nodes.intent_parser import intent_parser_node
from .nodes.researcher import researcher_node
from .nodes.writer_outline import writer_outline_node
from .nodes.writer_draft import writer_draft_node
from .nodes.editor import editor_node
# ── 干预节点 ──────────────────────────────────────────────────────────────────
async def pause_material_confirm(state: AIWritingState) -> dict:
"""PAUSE-1:素材确认"""
pkg = state["material_package"]
user_input: dict = interrupt({
"pause_point": "material_confirm",
"message": "素材收集专家已完成素材搜集,请确认素材后继续。",
"materials": pkg["materials"],
"keywords": pkg["keywords"],
"summary": pkg["summary"],
})
action = user_input.get("action", "confirm")
if action == "confirm":
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
updated_pkg = {**pkg, "materials": filtered}
return {
"material_package": updated_pkg,
"last_user_intervention": user_input,
"status": "writing_outline",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材,开始规划大纲"}],
}
# re_search:重新搜索
extra_kws = user_input.get("extraKeywords") or []
return {
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
"status": "researching",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
}
def _normalize_outline(outline: dict) -> dict:
"""将前端回传的 camelCase 大纲统一为后端 snake_case 格式。"""
if not outline:
return outline
sections = []
for s in outline.get("sections", []):
sections.append({
"section_title": s.get("section_title") or s.get("sectionTitle", ""),
"key_points": s.get("key_points") or s.get("keyPoints", []),
"material_ids": s.get("material_ids") or s.get("materialIds", []),
})
return {"title": outline.get("title", ""), "sections": sections}
async def pause_outline_confirm(state: AIWritingState) -> dict:
"""PAUSE-2:大纲确认"""
user_input: dict = interrupt({
"pause_point": "outline_confirm",
"message": "作家已完成大纲规划,请确认或编辑大纲后继续。",
"outline": state["current_outline"],
})
action = user_input.get("action", "confirm")
if action == "confirm":
raw_outline = user_input.get("editedOutline") or state["current_outline"]
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
return {
"current_outline": edited,
"last_user_intervention": user_input,
"status": "writing_draft",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}],
}
# re_outline
feedback = user_input.get("outlineFeedback", "")
return {
"last_user_intervention": {**user_input, "_feedback": feedback},
"status": "writing_outline",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
}
async def pause_draft_confirm(state: AIWritingState) -> dict:
"""PAUSE-3:草稿确认"""
draft = state["current_draft"]
user_input: dict = interrupt({
"pause_point": "draft_confirm",
"message": "草稿已完成,请确认或给出修改意见。",
"draft_title": draft["title"],
"revision_count": draft["revision_count"],
"review_result": state.get("review_result"),
})
action = user_input.get("action", "to_editor")
if action == "finalize":
return {
"last_user_intervention": user_input,
"status": "done",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户确认定稿"}],
}
if action == "user_revise":
notes = user_input.get("userRevisionNotes", "")
return {
"last_user_intervention": {**user_input, "_user_notes": notes},
"revision_count": state.get("revision_count", 0) + 1,
"status": "revising",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"按用户意见重写({notes[:30]})"}],
}
# to_editor
return {
"last_user_intervention": user_input,
"status": "reviewing",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "提交编辑审核"}],
}
async def pause_review_confirm(state: AIWritingState) -> dict:
"""PAUSE-4:编辑打回确认"""
user_input: dict = interrupt({
"pause_point": "review_confirm",
"message": "编辑认为文章需要修改,请决定下一步操作。",
"review_result": state["review_result"],
"revision_count": state.get("revision_count", 0),
"max_revisions": state.get("max_revisions", 3),
})
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
return {
"last_user_intervention": user_input,
"status": "done",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户强制定稿"}],
}
action = user_input.get("action", "accept_review")
extra_notes = user_input.get("userRevisionNotes", "")
return {
"last_user_intervention": {**user_input, "_extra_notes": extra_notes, "action": action},
"revision_count": state.get("revision_count", 0) + 1,
"status": "revising",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "按编辑意见重写"}],
}
# ── 条件路由 ──────────────────────────────────────────────────────────────────
def after_material_pause(state: AIWritingState) -> str:
return "researcher" if state["status"] == "researching" else "writer_outline"
def after_outline_pause(state: AIWritingState) -> str:
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
def after_draft_pause(state: AIWritingState) -> str:
status = state["status"]
if status == "done":
return "end"
if status == "revising":
return "writer_draft"
return "editor"
def after_review(state: AIWritingState) -> str:
review = state.get("review_result") or {}
if review.get("verdict") == "pass":
return "pause_draft"
rev_count = state.get("revision_count", 0)
max_rev = state.get("max_revisions", 3)
if rev_count >= max_rev:
return "pause_draft" # 超出上限,强制到草稿确认
return "pause_review"
def after_review_pause(state: AIWritingState) -> str:
return "end" if state["status"] == "done" else "writer_draft"
# ── 组装 Graph ────────────────────────────────────────────────────────────────
def build_ai_writing_graph():
g = StateGraph(AIWritingState)
g.add_node("intent_parser", intent_parser_node)
g.add_node("researcher", researcher_node)
g.add_node("pause_material", pause_material_confirm)
g.add_node("writer_outline", writer_outline_node)
g.add_node("pause_outline", pause_outline_confirm)
g.add_node("writer_draft", writer_draft_node)
g.add_node("pause_draft", pause_draft_confirm)
g.add_node("editor", editor_node)
g.add_node("pause_review", pause_review_confirm)
g.set_entry_point("intent_parser")
g.add_edge("intent_parser", "researcher")
g.add_edge("researcher", "pause_material")
g.add_conditional_edges("pause_material", after_material_pause,
{"researcher": "researcher", "writer_outline": "writer_outline"})
g.add_edge("writer_outline", "pause_outline")
g.add_conditional_edges("pause_outline", after_outline_pause,
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
g.add_edge("writer_draft", "pause_draft")
g.add_conditional_edges("pause_draft", after_draft_pause,
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
g.add_conditional_edges("editor", after_review,
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
g.add_conditional_edges("pause_review", after_review_pause,
{"end": END, "writer_draft": "writer_draft"})
checkpointer = MemorySaver()
return g.compile(checkpointer=checkpointer)
AI_WRITING_GRAPH = build_ai_writing_graph()
@@ -0,0 +1,242 @@
"""节点共用工具:JSON 解析容错、LLM 创建。
弱模型经常输出不规范 JSON(中文标点、结尾多余逗号、被 markdown 包裹、
夹带解释文字、嵌套对象)。parse_json 用多级降级策略尽量抢救出结构化结果。
"""
from __future__ import annotations
import json
import logging
import re
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
logger = logging.getLogger(__name__)
def get_model(config: RunnableConfig, model_name: str | None = None):
"""创建对话模型。
model_name 显式传入时优先使用(如素材收集专家的检索词/排序专用快模型),
否则回退到 configurable.model_name(用户在写作表单里选的主模型)。
"""
from deerflow.models import create_chat_model
name = model_name or config.get("configurable", {}).get("model_name", "") or None
return create_chat_model(name=name)
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
_FULLWIDTH_FIX = {
"“": '"', # 左双引号 “
"”": '"', # 右双引号 ”
"‘": "'", # 左单引号 ‘
"’": "'", # 右单引号 ’
":": ":", # 全角冒号 :
",": ",", # 全角逗号 ,
"(": "(", # 全角左括号 (
")": ")", # 全角右括号 )
"{": "{", # 全角左花括号 {
"}": "}", # 全角右花括号 }
"[": "[", # 全角左方括号 [
"]": "]", # 全角右方括号 ]
"【": "[", # 黑方括号 【
"】": "]", # 黑方括号 】
}
def _extract_balanced(text: str) -> str | None:
"""从文本中提取第一个括号平衡的 {...} 块(支持嵌套,忽略字符串内的括号)。"""
start = text.find("{")
if start == -1:
return None
depth = 0
in_str = False
escape = False
for i in range(start, len(text)):
ch = text[i]
if escape:
escape = False
continue
if ch == "\\":
escape = True
continue
if ch == '"':
in_str = not in_str
continue
if in_str:
continue
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return text[start:i + 1]
return None
def _repair_json(text: str) -> str:
"""尽力修复弱模型常见的 JSON 错误:全角标点、结尾多余逗号。"""
for bad, good in _FULLWIDTH_FIX.items():
text = text.replace(bad, good)
# 去掉对象/数组结尾多余的逗号: ,} 或 ,]
text = re.sub(r",(\s*[}\]])", r"\1", text)
return text
def _try_load(text: str) -> dict | None:
try:
result = json.loads(text)
except Exception:
return None
return result if isinstance(result, dict) else None
def parse_json(text: str, fallback: dict) -> dict:
"""从 LLM 响应中提取 JSON 对象,多级降级,失败时返回 fallback。"""
if not text or not text.strip():
return fallback
text = text.strip()
# 1. 直接解析
if (r := _try_load(text)) is not None:
return r
# 2. markdown 代码块内容
m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
if m:
fenced = m.group(1).strip()
if (r := _try_load(fenced)) is not None:
return r
text = fenced # 用代码块内容继续后续步骤
# 3. 平衡括号提取(处理夹带解释文字、嵌套对象的情况)
candidate = _extract_balanced(text)
if candidate:
if (r := _try_load(candidate)) is not None:
return r
# 4. 修复后再试
if (r := _try_load(_repair_json(candidate))) is not None:
return r
# 5. 整体修复后兜底
if (r := _try_load(_repair_json(text))) is not None:
return r
logger.warning("JSON parse failed, using fallback. raw=%r", text[:200])
return fallback
def _content_to_text(content) -> str:
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
if isinstance(content, str):
return content
if isinstance(content, list):
return "".join(
part.get("text", "") if isinstance(part, dict) else str(part)
for part in content
)
return str(content)
# 中文常见标点:参与「字数」统计,与 Word 字数统计 / 用户直觉口径对齐
_CJK_PUNCTUATION = ",。、;:?!“”‘’()《》【】「」『』〈〉—…·~"
def count_words(text: str) -> int:
"""统计中文语境下的「字数」。
口径:汉字逐字计 + 中文标点逐个计 + 连续数字/英文各算 1 字。
不计入空白、markdown 语法符号(# * - 等)与 ASCII 标点。
前后端共用同一口径(前端 utils/wordCount.ts 实现一致)。
"""
if not text:
return 0
cjk = len(re.findall(r"[一-鿿]", text))
cjk_punct = sum(1 for c in text if c in _CJK_PUNCTUATION)
alnum_runs = len(re.findall(r"[A-Za-z0-9]+", text))
return cjk + cjk_punct + alnum_runs
class ThinkTagParser:
"""流式 <think>...</think> 标签解析器。
模型可能在任意 chunk 边界截断标签,通过尾部缓冲处理跨 chunk 边界情况。
调用 feed(chunk) 返回 list[tuple[bool, str]],每项为 (is_thinking, text)。
"""
_OPEN_TAG = "<think>"
_CLOSE_TAG = "</think>"
def __init__(self) -> None:
self._in_think = False
self._buf = "" # 潜在标签的部分匹配缓冲
def feed(self, chunk: str) -> list[tuple[bool, str]]:
"""处理一个流式 chunk,返回 (is_thinking, text) 片段列表。"""
self._buf += chunk
result: list[tuple[bool, str]] = []
while self._buf:
if self._in_think:
tag = self._CLOSE_TAG
idx = self._buf.find(tag)
if idx != -1:
if idx > 0:
result.append((True, self._buf[:idx]))
self._buf = self._buf[idx + len(tag):]
self._in_think = False
else:
partial = self._partial_suffix(self._buf, tag)
if partial:
result.append((True, self._buf[:-partial]))
self._buf = self._buf[-partial:]
else:
result.append((True, self._buf))
self._buf = ""
break
else:
tag = self._OPEN_TAG
idx = self._buf.find(tag)
if idx != -1:
if idx > 0:
result.append((False, self._buf[:idx]))
self._buf = self._buf[idx + len(tag):]
self._in_think = True
else:
partial = self._partial_suffix(self._buf, tag)
if partial:
result.append((False, self._buf[:-partial]))
self._buf = self._buf[-partial:]
else:
result.append((False, self._buf))
self._buf = ""
break
return [(is_t, t) for (is_t, t) in result if t]
@staticmethod
def _partial_suffix(text: str, tag: str) -> int:
"""返回 text 尾部与 tag 前缀匹配的最长长度(>0 说明可能是部分标签)。"""
max_check = min(len(tag) - 1, len(text))
for length in range(max_check, 0, -1):
if tag.startswith(text[-length:]):
return length
return 0
async def llm_json(system: str, user: str, fallback: dict, config: RunnableConfig, model_name: str | None = None) -> dict:
"""调用 LLM,解析 JSON 输出,失败用 fallback。
model_name 可覆盖默认模型(如检索词/排序使用配置的专用快模型)。
"""
messages = [SystemMessage(content=system), HumanMessage(content=user)]
try:
# get_model 放进 try:模型名在 config.yaml 找不到时 create_chat_model 会抛
# ValueError,若留在 try 外会逐层上抛使整个图硬崩溃。
model = get_model(config, model_name=model_name)
resp = await model.ainvoke(messages)
return parse_json(_content_to_text(resp.content), fallback)
except Exception as e:
logger.error("llm_json failed: %s", e, exc_info=True)
return {**fallback, "_llm_error": str(e)}
@@ -0,0 +1,180 @@
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
from __future__ import annotations
import asyncio
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, ReviewResult, ReviewIssue
from ..prompts.editor_prompts import (
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
logger = logging.getLogger(__name__)
_DEFAULT_PASS_THRESHOLD = 75
def _materials_summary(pkg: dict) -> str:
lines = []
for m in (pkg.get("materials") or [])[:6]:
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:200]}")
return "\n".join(lines) or "(无素材)"
def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
issues = []
for item in (raw_issues or []):
issues.append(ReviewIssue(
category=category,
severity=item.get("severity", "minor"),
location=item.get("location", ""),
issue=item.get("issue", ""),
suggestion=item.get("suggestion", ""),
))
return issues
async def _stream_one_review(
model, user_prompt: str, category: str,
sse_queue: asyncio.Queue | None, session_id: str,
) -> str:
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
parser = ThinkTagParser()
async for chunk in model.astream([
SystemMessage(content=EDITOR_SYSTEM),
HumanMessage(content=user_prompt),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("review_chunk", {
"session_id": session_id,
"category": category,
"text": text,
"is_thinking": is_thinking,
}))
return full_text
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
draft = state.get("current_draft")
if not draft:
return {
"status": "error",
"progress_events": [{"type": "error", "agent_name": "编辑", "message": "草稿为空,无法审核"}],
}
article = draft["full_markdown"]
pkg = state.get("material_package") or {}
mats = _materials_summary(pkg)
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
REVIEW_TASKS = [
("fact", FACT_REVIEW_USER.format(materials_summary=mats, article=article)),
("logic", LOGIC_REVIEW_USER.format(article=article)),
("language", LANGUAGE_REVIEW_USER.format(article=article)),
]
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
except Exception as e:
logger.error("editor model creation failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error", "agent_name": "编辑",
"title": "模型错误", "detail": str(e), "message": "审核失败",
}],
}
results: dict[str, dict] = {}
review_errors: list[str] = [] # 分项审查的模型异常,供完成后非致命提示
_CATEGORY_LABEL = {"fact": "事实性", "logic": "逻辑性", "language": "语言文字"}
for category, user_prompt in REVIEW_TASKS:
fallback = {"score": 80, "issues": []}
try:
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
res = parse_json(full_text, fallback)
except Exception as e:
logger.warning("editor review [%s] failed: %s", category, e)
res = fallback
review_errors.append(f"「{_CATEGORY_LABEL.get(category, category)}审查」{type(e).__name__}: {e}")
results[category] = res
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
if sse_queue is not None:
issues = _parse_issues(res.get("issues", []), category)
await sse_queue.put(("review_item_done", {
"session_id": session_id,
"category": category,
"score": int(res.get("score", 80)),
"issues": [dict(iss) for iss in issues],
}))
else:
logger.debug("editor [%s] done (no sse_queue)", category)
# 汇总
fact_score = int(results["fact"].get("score", 80))
logic_score = int(results["logic"].get("score", 80))
lang_score = int(results["language"].get("score", 80))
overall_score = (fact_score + logic_score + lang_score) // 3
all_issues: list[ReviewIssue] = (
_parse_issues(results["fact"].get("issues", []), "fact") +
_parse_issues(results["logic"].get("issues", []), "logic") +
_parse_issues(results["language"].get("issues", []), "language")
)
all_issues.sort(key=lambda x: (0 if x["severity"] == "major" else 1))
verdict = "pass" if overall_score >= pass_threshold else "reject"
review = ReviewResult(
verdict=verdict,
fact_score=fact_score,
logic_score=logic_score,
language_score=lang_score,
overall_score=overall_score,
pass_threshold=pass_threshold,
revision_notes=all_issues,
)
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
notices: list[dict] = []
if review_errors:
notices.append({
"type": "notice",
"agent_name": "编辑",
"title": "模型错误",
"detail": "部分审查项调用失败,已按默认分兜底。原因:" + ";".join(review_errors[:3]),
})
return {
"review_result": review,
"progress_events": [
{"type": "step_start", "agent_name": "编辑", "message": "开始审核"},
*notices,
{
"type": "review_ready",
"agent_name": "编辑",
"message": f"审核完成,综合得分 {overall_score},结论:{'通过' if verdict == 'pass' else '打回修改'}",
},
],
}
@@ -0,0 +1,131 @@
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
from __future__ import annotations
import re
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState
from ._utils import llm_json
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
_DEFAULT_ARTICLE_TYPES: list[dict] = [
{"key": "news", "label": "新闻资讯"},
{"key": "analysis", "label": "分析报道"},
{"key": "feature", "label": "深度特稿"},
{"key": "report", "label": "调研报告"},
]
_SYSTEM_TEMPLATE = """你是一名写作助手。
你的任务:分析用户的写作意图,提取结构化的写作配置。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- article_type 只能是以下之一:{type_options}。
【完整示例】
输入意图:帮我写一篇面向行业专家的深度文章,分析今年新能源汽车出口为什么大涨,大概两千字
正确输出:
{{"article_type": "feature", "target_audience": "行业专家", "word_count_target": 2000, "refined_intent": "分析今年新能源汽车出口高速增长的原因"}}"""
_USER_TEMPLATE = """用户写作意图:{user_intent}
请分析并输出以下 JSON(如果意图中未明确某字段,给出合理默认值):
{{{{
"article_type": "{type_options} 之一",
"target_audience": "目标读者描述",
"word_count_target": 800,
"refined_intent": "精炼后的写作意图(一句话)"
}}}}"""
def _build_type_options(types: list[dict]) -> str:
"""将类型列表格式化为提示词中的枚举说明,如 news(新闻资讯)/ analysis(分析报道)。"""
return " / ".join(f'{t["key"]}({t["label"]})' for t in types)
def _coerce_word_count(raw) -> int | None:
"""稳健地把模型返回的字数字段转成正整数;无法解析时返回 None。
弱模型(尤其老模型被陌生词带偏时)常返回「约800字」「两千」「800.0」
这类非纯数字形式,直接 int() 会抛 ValueError/TypeError 使整个图崩溃。
此处用「提取首个数字串」策略容错,解析不出就返回 None 让调用方保留默认值。
"""
if raw is None:
return None
if isinstance(raw, bool): # bool 是 int 子类,需先排除
return None
if isinstance(raw, (int, float)):
value = int(raw)
return value if value > 0 else None
if not isinstance(raw, str):
# dict / list 等结构化类型:str() 化后可能误含数字,直接判为无法解析
return None
m = re.search(r"\d[\d,]*", raw)
if not m:
return None
try:
value = int(m.group().replace(",", ""))
except ValueError:
return None
return value if value > 0 else None
async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> dict:
# 从 configurable 读取动态类型列表,未配置时退回默认值
configurable = (config or {}).get("configurable", {})
valid_article_types: list[dict] = configurable.get("valid_article_types") or _DEFAULT_ARTICLE_TYPES
valid_keys: list[str] = [t["key"] for t in valid_article_types]
default_key: str = valid_keys[0] if valid_keys else "analysis"
type_options = _build_type_options(valid_article_types)
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
user = _USER_TEMPLATE.format(
user_intent=state["user_intent"],
type_options=" | ".join(valid_keys),
)
fallback_type = state.get("article_type", default_key)
if fallback_type not in valid_keys:
fallback_type = default_key
result = await llm_json(
system,
user,
fallback={
"article_type": fallback_type,
"target_audience": state.get("target_audience", "普通读者"),
"word_count_target": state.get("word_count_target", 800),
"refined_intent": state["user_intent"],
},
config=config,
)
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
# 流程仍可继续,但用户需知道结构化配置可能不准)。
progress_events: list[dict] = []
if result.get("_llm_error"):
progress_events.append({
"type": "notice",
"agent_name": "系统",
"title": "模型错误",
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
})
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
updates: dict = {"status": "researching", "progress_events": progress_events}
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
raw_type = result.get("article_type", "")
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
if result.get("target_audience"):
updates["target_audience"] = result["target_audience"]
word_count = _coerce_word_count(result.get("word_count_target"))
if word_count is not None:
updates["word_count_target"] = word_count
if result.get("refined_intent"):
updates["user_intent"] = result["refined_intent"]
return updates
@@ -0,0 +1,198 @@
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
from __future__ import annotations
import asyncio
import logging
import uuid
from typing import List
from langchain_core.runnables import RunnableConfig
from ..search import get_search_provider, SearchResult
from ..state import AIWritingState, Material, MaterialPackage
from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
)
from ._utils import llm_json
logger = logging.getLogger(__name__)
_MAX_MATERIALS = 12
def _notice(title: str, detail: str, raw: str = "") -> dict:
"""构造一个非致命告警事件,前端在时间线显示「标题 + 真实原因」,不打断写作流程。
raw 非空时附带接口的原始错误响应体,前端默认折叠展示。
"""
event = {"type": "notice", "agent_name": "素材收集专家", "title": title, "detail": detail}
if raw:
event["raw"] = raw
return event
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
intent = state["user_intent"]
article_type = state.get("article_type", "analysis")
target_audience = state.get("target_audience", "普通读者")
# 检查是否是补充搜索(PAUSE-1 re_search action)
intervention = state.get("last_user_intervention") or {}
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
cfg = config.get("configurable", {})
sse_queue = cfg.get("sse_queue")
sse_session_id: str = cfg.get("sse_session_id", "")
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
rank_model: str | None = ai_writing_config.get("rank_model") or None
async def _emit(message: str) -> None:
if sse_queue is None:
return
await sse_queue.put(("research_progress", {
"session_id": sse_session_id,
"message": message,
}))
# Step 1: 提取关键词
await _emit("正在生成检索词…")
kw_result = await llm_json(
EXTRACT_KEYWORDS_SYSTEM,
EXTRACT_KEYWORDS_USER.format(
user_intent=intent,
article_type=article_type,
target_audience=target_audience,
),
fallback={"keywords": [intent]},
config=config,
model_name=keyword_model,
)
keywords: list[str] = kw_result.get("keywords", [intent])
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
# Step 2: 并发搜索
provider = get_search_provider(ai_writing_config)
search_keywords = keywords[:6]
total = len(search_keywords)
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
completed = 0
async def _search_one(kw: str) -> list[SearchResult]:
nonlocal completed
try:
return await provider.search(kw, max_results=3)
finally:
completed += 1
if completed < total:
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
else:
await _emit(f"已完成全部 {total} 次检索")
search_tasks = [_search_one(kw) for kw in search_keywords]
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
# 合并去重
seen_urls: set[str] = set()
raw_materials: list[Material] = []
existing_pkg = state.get("material_package")
if existing_pkg and intervention.get("action") != "re_search":
for m in existing_pkg.get("materials", []):
seen_urls.add(m.get("url", ""))
raw_materials.append(m)
search_errors: list[str] = []
for results in all_results_nested:
if isinstance(results, Exception):
logger.warning("search failed: %s", results)
search_errors.append(f"{type(results).__name__}: {results}")
continue
for r in results:
if r.url and r.url in seen_urls:
continue
seen_urls.add(r.url)
raw_materials.append(Material(
id=f"m{len(raw_materials)+1:03d}",
title=r.title,
content=r.content[:500],
source=r.source,
published_at=r.published_at,
relevance_score=r.relevance_score,
source_type=r.source_type,
url=r.url,
recUuid=r.recUuid,
))
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
search_errors.extend(getattr(provider, "errors", []))
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
await _emit("检索完成,正在筛选排序素材…")
rank_error = ""
if len(raw_materials) > _MAX_MATERIALS:
materials_list = "\n".join(
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
for m in raw_materials
)
rank_result = await llm_json(
RANK_MATERIALS_SYSTEM,
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
fallback={"ranked_ids": [m["id"] for m in raw_materials[:_MAX_MATERIALS]], "summary": f"共搜索到{len(raw_materials)}条素材"},
config=config,
model_name=rank_model,
)
rank_error = rank_result.get("_llm_error", "")
ranked_ids: list[str] = rank_result.get("ranked_ids", [m["id"] for m in raw_materials])
id_to_mat = {m["id"]: m for m in raw_materials}
sorted_materials = [id_to_mat[mid] for mid in ranked_ids if mid in id_to_mat]
# 加入未被排序的素材
sorted_ids_set = set(ranked_ids)
for m in raw_materials:
if m["id"] not in sorted_ids_set:
sorted_materials.append(m)
materials = sorted_materials[:_MAX_MATERIALS]
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
else:
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:_MAX_MATERIALS]
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
if not materials:
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
notices: list[dict] = []
if kw_result.get("_llm_error"):
notices.append(_notice("模型错误", f"检索词生成失败,已用默认关键词兜底。原因:{kw_result['_llm_error']}"))
if rank_error:
notices.append(_notice("模型错误", f"素材排序失败,已按相关度兜底排序。原因:{rank_error}"))
if search_errors:
uniq = list(dict.fromkeys(search_errors))
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
notices.append(_notice(
"ES 波动导致检索异常",
";".join(uniq[:3]),
raw="\n\n----------\n\n".join(raw_dumps[:3]),
))
return {
"material_package": pkg,
"status": "awaiting_material_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
*notices,
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
],
}
@@ -0,0 +1,234 @@
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, DraftArticle
from ..prompts.writer_prompts import DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
logger = logging.getLogger(__name__)
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
lines = []
for mid in material_ids:
if mid in mat_map:
m = mat_map[mid]
lines.append(f"[{mid}] {m['title']}: {m['content'][:300]}")
if not lines:
for m in pkg.get("materials", [])[:2]:
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:300]}")
return "\n".join(lines) if lines else "(无相关素材)"
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
outline = state.get("current_outline")
if not outline:
return {
"status": "error",
"progress_events": [{"type": "error", "agent_name": "作家", "message": "大纲为空,无法写作"}],
}
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
revision_count = state.get("revision_count", 0)
# SSE 流式推送所需
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
# 修改意见(来自用户或编辑)
revision_notes = ""
if intervention.get("action") == "user_revise":
notes = intervention.get("_user_notes") or intervention.get("user_revision_notes", "")
if notes:
revision_notes = f"\n用户修改意见:{notes}"
elif intervention.get("action") in ("accept_review", "custom_revise"):
review = state.get("review_result") or {}
issues = review.get("revision_notes", [])
extra = intervention.get("_extra_notes") or intervention.get("user_revision_notes", "")
parts = []
for iss in issues[:5]:
parts.append(f"- [{iss.get('severity','minor')}] {iss.get('location','')}: {iss.get('suggestion','')}")
if extra:
parts.append(f"- 用户补充:{extra}")
if parts:
revision_notes = "\n修改要求:\n" + "\n".join(parts)
word_count_target = state.get("word_count_target", 800)
sections = outline.get("sections", [])
n_sections = max(len(sections), 1)
# 单节硬下限:总目标 / 节数,乘 1.1 过量系数补偿模型对短目标的天然欠量;单节至少 150 字
words_per_section = max(150, int(word_count_target * 1.1 / n_sections))
article_title = outline.get("title", "文章")
# 兼容后端 snake_case 与前端编辑后回传的 camelCase
def _section_title(idx: int, sec: dict) -> str:
return sec.get("section_title") or sec.get("sectionTitle") or f"章节{idx + 1}"
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
except Exception as e:
logger.error("draft model creation failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error", "agent_name": "作家",
"title": "模型错误", "detail": str(e), "message": "草稿生成失败",
}],
}
section_titles: list[str] = []
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
for i, section in enumerate(sections):
sec_title = _section_title(i, section)
key_points = section.get("key_points") or section.get("keyPoints") or []
mat_ids = section.get("material_ids") or section.get("materialIds") or []
section_materials = _get_section_materials(mat_ids, pkg)
key_points_str = "\n".join(f"- {p}" for p in key_points)
# 文章整体结构:让作家看到全文章节与自己的定位,避免章节间内容重复、引言结论脱节
outline_context = "\n".join(
f"{j + 1}. {_section_title(j, s)}" + (" ← 当前正在写这一章" if j == i else "")
for j, s in enumerate(sections)
)
user_msg = DRAFT_SECTION_USER.format(
article_title=article_title,
outline_context=outline_context,
section_title=sec_title,
key_points=key_points_str,
section_materials=section_materials,
revision_section=revision_notes,
target_words=words_per_section,
)
section_buf = "" # 只累积正文(非思考)用于拼接 Markdown
parser = ThinkTagParser()
try:
async for chunk in model.astream([
SystemMessage(content=DRAFT_SECTION_SYSTEM),
HumanMessage(content=user_msg),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
section_buf += text
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": text,
"is_thinking": is_thinking,
"is_section_done": False,
}))
# 章节完成
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": "",
"is_section_done": True,
}))
section_titles.append(sec_title)
section_bufs.append(section_buf.strip())
except Exception as e:
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
section_titles.append(sec_title)
section_bufs.append("(内容生成失败,请重试)")
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
def _assemble() -> str:
body = "\n\n".join(f"## {t}\n\n{b}" for t, b in zip(section_titles, section_bufs))
return f"# {article_title}\n\n{body}"
full_markdown = _assemble()
word_count = count_words(full_markdown)
# 扩写兜底:总字数不足目标 90% 时,重写并扩充最短的章节(单轮,非流式)
if section_bufs and word_count < word_count_target * 0.9:
idx = min(range(len(section_bufs)), key=lambda k: count_words(section_bufs[k]))
target_sec = sections[idx]
expand_target = count_words(section_bufs[idx]) + (word_count_target - word_count)
try:
expand_msg = DRAFT_EXPAND_USER.format(
article_title=article_title,
section_title=section_titles[idx],
key_points="\n".join(
f"- {p}" for p in (target_sec.get("key_points") or target_sec.get("keyPoints") or [])
),
section_materials=_get_section_materials(
target_sec.get("material_ids") or target_sec.get("materialIds") or [], pkg
),
existing_text=section_bufs[idx],
target_words=expand_target,
)
resp = await model.ainvoke([
SystemMessage(content=DRAFT_SECTION_SYSTEM),
HumanMessage(content=expand_msg),
])
parser = ThinkTagParser()
expanded = "".join(
t for is_thinking, t in parser.feed(_content_to_text(resp.content)) if not is_thinking
).strip()
# 仅在扩写确实更长时才采用,避免越改越短
if count_words(expanded) > count_words(section_bufs[idx]):
section_bufs[idx] = expanded
full_markdown = _assemble()
word_count = count_words(full_markdown)
except Exception as e:
logger.warning("draft expand failed for section [%s]: %s", section_titles[idx], e)
draft = DraftArticle(
title=outline.get("title", "文章"),
outline=outline,
full_markdown=full_markdown,
word_count=word_count,
revision_count=revision_count,
)
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
notices: list[dict] = []
if section_errors:
notices.append({
"type": "notice",
"agent_name": "作家",
"title": "模型错误",
"detail": "部分章节生成失败,对应段落已占位。原因:" + ";".join(section_errors[:3]),
})
return {
"current_draft": draft,
"drafts": [draft],
"status": "awaiting_draft_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
*notices,
{
"type": "draft_ready",
"agent_name": "作家",
"message": f"草稿完成,共 {word_count} 字",
"draft_title": outline.get("title", "文章"),
"draft_markdown": full_markdown,
"revision_count": revision_count,
},
],
}
@@ -0,0 +1,121 @@
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, Outline, OutlineSection
from ..prompts.writer_prompts import OUTLINE_SYSTEM, OUTLINE_USER
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
logger = logging.getLogger(__name__)
# 上下文压缩:传给大纲规划的素材摘要做条数和单条长度双重截断,防止 token 溢出。
_MAX_OUTLINE_MATERIALS = 12
_MATERIAL_SNIPPET_CHARS = 150
def _materials_summary(pkg: dict) -> str:
lines = []
for m in pkg.get("materials", [])[:_MAX_OUTLINE_MATERIALS]:
lines.append(f"[{m['id']}] {m['title']} | {m['source']} | {m['content'][:_MATERIAL_SNIPPET_CHARS]}")
return "\n".join(lines) or "(无素材)"
def _parse_outline(raw: dict) -> Outline:
sections = []
for s in raw.get("sections", []):
sections.append(OutlineSection(
section_title=s.get("section_title", ""),
key_points=s.get("key_points", []),
material_ids=s.get("material_ids", []),
))
if not sections:
sections = [OutlineSection(section_title="正文", key_points=["主要内容"], material_ids=[])]
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
feedback_section = ""
if intervention.get("action") == "re_outline" and intervention.get("_feedback"):
feedback_section = f"\n重新规划要求:{intervention['_feedback']}"
elif intervention.get("outline_feedback"):
feedback_section = f"\n重新规划要求:{intervention['outline_feedback']}"
user_prompt = OUTLINE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
materials_summary=_materials_summary(pkg),
feedback_section=feedback_section,
)
# SSE 流式推送
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
full_text = "" # 只累积正文(非思考)用于 JSON 解析
parser = ThinkTagParser()
try:
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
model = get_model(config)
async for chunk in model.astream([
SystemMessage(content=OUTLINE_SYSTEM),
HumanMessage(content=user_prompt),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("outline_chunk", {
"session_id": session_id,
"text": text,
"is_thinking": is_thinking,
}))
except Exception as e:
logger.error("outline streaming failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error",
"agent_name": "作家",
"title": "模型错误",
"detail": str(e),
"message": "大纲生成失败",
}],
}
fallback = {
"title": "未命名文章",
"sections": [
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
],
}
result = parse_json(full_text, fallback)
outline = _parse_outline(result)
section_count = len(outline["sections"])
return {
"current_outline": outline,
"status": "awaiting_outline_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": "开始规划大纲"},
{"type": "outline_ready", "agent_name": "作家", "message": f"大纲已生成,共 {section_count} 个章节"},
],
}
@@ -0,0 +1,52 @@
EDITOR_SYSTEM = """你是一名资深主编(编辑),标准极高,要求严格。
你的目标:审查文章质量,给出客观评分和具体修改意见。
你的工作背景:你在一家权威媒体担任主编十五年,见过各种质量的文章。
你只能做以下事情:
1. 从指定维度审查文章
2. 给出该维度的评分(0-100 的整数)和具体问题列表
你不能改写文章,只做当前指定维度的审查。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- score 是 0-100 的整数。
- 每个问题的 severity 必须是 "major"(必须修改)或 "minor"(建议修改)。
- 文章在该维度没有问题时,issues 填空数组 []。
【完整示例】
正确输出:
{
"score": 82,
"issues": [
{"severity": "major", "location": "第二段", "issue": "引用的增长数据没有素材支撑", "suggestion": "补充素材 m002 中的具体数据,并注明来源"},
{"severity": "minor", "location": "结尾段", "issue": "结论略显仓促", "suggestion": "增加一句对未来行业趋势的判断"}
]
}"""
FACT_REVIEW_USER = """请审查以下文章的【事实准确性】维度。
判断标准:引用的数据/事实是否有素材支撑,是否有捏造或夸大。
参考素材:
{materials_summary}
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
LOGIC_REVIEW_USER = """请审查以下文章的【逻辑结构】维度。
判断标准:论点是否清晰,论据是否充分,段落是否连贯,结构是否合理。
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
LANGUAGE_REVIEW_USER = """请审查以下文章的【语言文字】维度。
判断标准:语言是否流畅,用词是否准确,有无错别字或病句。
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
@@ -0,0 +1,59 @@
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:为写作任务提取搜索关键词。
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
你只能做以下事情:
1. 从写作意图中提取 3-6 个搜索关键词
你不能写文章,不能评价内容好坏,只做关键词提取。
【严禁思考】
- 这是一个简单的结构化抽取任务,禁止任何思考、分析、推理或规划过程。
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 ",不要用中文引号 “”。
- 关键词要具体、可搜索,避免过于宽泛的词。
【完整示例】
输入意图:写一篇分析今年新能源汽车出口增长的文章
正确输出:
{"keywords": ["新能源汽车出口", "汽车出口数据", "新能源汽车海外市场", "电动车出口增长", "汽车出口政策"]}"""
EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
请提取 3-6 个搜索关键词,用于搜集相关素材。只输出 JSON:
{{"keywords": ["关键词1", "关键词2"]}}"""
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:对搜索结果按相关度排序和筛选。
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
你只能做以下事情:
1. 从提供的素材列表中,选出与写作意图最相关的素材ID,按相关度从高到低排序
你不能写文章,只做素材筛选排序。
【严禁思考】
- 这是一个简单的筛选排序任务,禁止任何思考、分析、推理或规划过程。
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- ranked_ids 中的 ID 必须来自素材列表,不要编造 ID。
【完整示例】
正确输出:
{"ranked_ids": ["m003", "m001", "m005"], "summary": "共搜索到 8 条素材,精选 5 条最相关的"}"""
RANK_MATERIALS_USER = """写作意图:{user_intent}
素材列表(每条含ID和摘要):
{materials_list}
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}"""
@@ -0,0 +1,75 @@
OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据素材和意图,规划清晰的文章大纲。
你的工作背景:你有十年内容创作经验,擅长将零散素材组织成有逻辑的文章结构。
你只能做以下事情:
1. 分析素材,提取核心观点
2. 规划文章章节结构和每节要点
你不能写正文,只写大纲。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 ",不要用中文引号 “”。
- 规划 3-5 个章节;material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
【完整示例】
正确输出:
{
"title": "新能源汽车出口为何高速增长",
"sections": [
{"section_title": "引言:出口数据创新高", "key_points": ["今年出口量同比增长", "在全球市场的排名变化"], "material_ids": ["m001"]},
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
{"section_title": "结论与展望", "key_points": ["未来增长面临的挑战", "行业趋势判断"], "material_ids": ["m004"]}
]
}"""
OUTLINE_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
目标字数:{word_count_target}字{feedback_section}
可用素材摘要:
{materials_summary}
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据大纲和素材,写出一段高质量的文章内容。
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
你只能做以下事情:
1. 根据给定的章节要点和素材写这一段内容
2. 观点必须来自素材,不编造数据
你不能修改大纲,不能写其他章节,只写当前这一段。
【输出要求】
- 直接返回这一段的正文文字,不要 JSON,不要章节标题,不要解释。
- 不要写"以下是正文""本段内容如下"之类的开场白。
- 段落要连贯成文,不要罗列要点。
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
- 充分展开每个要点:每个要点至少用 2-3 句话论述,结合素材中的事实与数据说理,写满要求的篇幅。"""
DRAFT_SECTION_USER = """文章标题:{article_title}
文章整体结构:
{outline_context}
当前章节:{section_title}
章节要点:{key_points}
相关素材:
{section_materials}
{revision_section}
请写出「{section_title}」这一章节的正文内容,不少于 {target_words} 字,论述充分、内容充实。"""
DRAFT_EXPAND_USER = """文章标题:{article_title}
当前章节:{section_title}
章节要点:{key_points}
相关素材:
{section_materials}
这一章节已有内容如下:
{existing_text}
以上内容篇幅不足。请在保留原意和事实的基础上重写并扩充这一章节,使正文不少于 {target_words} 字。
要求:补充更多来自素材的事实、数据与分析,论述更充分、更具体;直接返回扩充后的完整章节正文,不要章节标题、不要解释。"""
@@ -0,0 +1,64 @@
from .base import BaseSearchProvider, SearchResult
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
"""
config.yaml 中配置:
ai_writing:
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
intranet_knowledge_base: samples
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
"""
cfg = ai_writing_config or {}
provider = cfg.get("search_provider", "internet")
if provider == "intranet":
return IntranetSearchProvider(
base_url=cfg.get("intranet_search_url", ""),
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
)
if provider == "tavily":
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
if provider == "demo":
return DemoSearchProvider()
# 默认 internet 模式:DuckDuckGo + demo fallback
region = cfg.get("ddg_region", "cn-zh")
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
class _FallbackSearchProvider(BaseSearchProvider):
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
self._primary = primary
self._fallback = fallback
@property
def errors(self) -> list[str]:
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
async def search(self, query: str, max_results: int = 5, **kwargs):
results = await self._primary.search(query, max_results=max_results, **kwargs)
if not results:
results = await self._fallback.search(query, max_results=max_results, **kwargs)
return results
async def health_check(self) -> bool:
return await self._primary.health_check() or await self._fallback.health_check()
__all__ = [
"BaseSearchProvider", "SearchResult",
"InternetSearchProvider", "IntranetSearchProvider",
"DemoSearchProvider", "DuckDuckGoSearchProvider",
"get_search_provider",
]
@@ -0,0 +1,40 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List
@dataclass
class SearchResult:
"""统一搜索结果格式 —— 内外网共用"""
title: str
content: str
source: str
published_at: str
url: str = ""
relevance_score: float = 0.0
source_type: str = "internet"
recUuid: str = ""
extra_metadata: dict = field(default_factory=dict)
class BaseSearchProvider(ABC):
"""搜索提供者抽象基类。
对接内网步骤:
1. 继承此类,创建 IntranetSearchProvider
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
4. 无需修改任何 Agent 代码
"""
@abstractmethod
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]: ...
@abstractmethod
async def health_check(self) -> bool: ...
@@ -0,0 +1,183 @@
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
from __future__ import annotations
import hashlib
from typing import List
from .base import BaseSearchProvider, SearchResult
# 演示素材库:按话题分类,用于关键词模糊匹配
_DEMO_CORPUS = [
{
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
"items": [
{
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
"source": "中国汽车工业协会",
"published_at": "2025-10-15",
"relevance_score": 0.95,
},
{
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
"source": "电池中国网",
"published_at": "2025-09-20",
"relevance_score": 0.88,
},
{
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
"source": "中国充电联盟",
"published_at": "2025-08-30",
"relevance_score": 0.82,
},
],
},
{
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
"items": [
{
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
"source": "中国信通院",
"published_at": "2025-10-08",
"relevance_score": 0.93,
},
{
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
"source": "麦肯锡中国",
"published_at": "2025-09-12",
"relevance_score": 0.87,
},
{
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
"source": "艾瑞咨询",
"published_at": "2025-10-01",
"relevance_score": 0.79,
},
],
},
{
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
"items": [
{
"title": "光伏装机量突破900GW,度电成本降至0.18元",
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
"source": "国家能源局",
"published_at": "2025-10-12",
"relevance_score": 0.92,
},
{
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
"source": "上海环境能源交易所",
"published_at": "2025-09-28",
"relevance_score": 0.85,
},
],
},
{
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
"items": [
{
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
"source": "医学界",
"published_at": "2025-09-18",
"relevance_score": 0.91,
},
],
},
{
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
"items": [
{
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
"source": "国家统计局",
"published_at": "2025-10-18",
"relevance_score": 0.90,
},
{
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
"source": "商务部",
"published_at": "2025-10-05",
"relevance_score": 0.84,
},
],
},
]
# 通用兜底素材(与任意查询都能凑合搭配)
_FALLBACK_ITEMS = [
{
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
"source": "产业研究院",
"published_at": "2025-08-15",
"relevance_score": 0.55,
},
{
"title": "政策解读:2025年产业发展专项支持资金管理办法",
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
"source": "工信部",
"published_at": "2025-07-20",
"relevance_score": 0.50,
},
{
"title": "市场竞争格局分析:头部集中与长尾分散并存",
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
"source": "普华永道",
"published_at": "2025-09-01",
"relevance_score": 0.48,
},
]
def _score_topic(query: str, keywords: list[str]) -> int:
"""简单关键词匹配打分"""
q = query.lower()
return sum(1 for kw in keywords if kw in q)
class DemoSearchProvider(BaseSearchProvider):
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
仅用于开发测试;生产环境应配置 intranet 提供者。
"""
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
items = []
if best_score > 0 and best_topic:
items.extend(best_topic["items"])
# 用通用素材补齐
for fb in _FALLBACK_ITEMS:
if len(items) >= max_results:
break
items.append(fb)
results = []
for i, it in enumerate(items[:max_results]):
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
results.append(SearchResult(
title=it["title"],
content=it["content"],
source=it["source"],
published_at=it["published_at"],
url=f"https://demo.internal/{url_hash}",
relevance_score=it["relevance_score"],
source_type="demo",
recUuid=f"demo-{i:03d}",
))
return results
async def health_check(self) -> bool:
return True
@@ -0,0 +1,81 @@
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
from __future__ import annotations
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class DuckDuckGoSearchProvider(BaseSearchProvider):
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
def __init__(self, region: str = "cn-zh", max_timeout: int = 30):
self.region = region
self.max_timeout = max_timeout
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
self.errors: list[str] = []
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(
self._sync_search, query, max_results
)
return results
except Exception as e:
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs not installed. Run: uv add ddgs")
self.errors.append("ImportError: ddgs 未安装")
return []
ddgs = DDGS(timeout=self.max_timeout)
try:
raw = ddgs.text(
query,
region=self.region,
safesearch="moderate",
max_results=max_results,
)
items = list(raw) if raw else []
except Exception as e:
logger.warning("DDGS.text failed: %s", e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
results = []
for item in items:
url = item.get("href", item.get("link", ""))
source = url.split("/")[2] if url and "/" in url else "web"
results.append(SearchResult(
title=item.get("title", ""),
content=item.get("body", item.get("snippet", ""))[:500],
source=source,
published_at=item.get("published", ""),
url=url,
relevance_score=0.75,
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return len(results) > 0
except Exception:
return False
@@ -0,0 +1,66 @@
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class InternetSearchProvider(BaseSearchProvider):
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(self._sync_search, query, max_results)
return results
except Exception as e:
logger.warning("internet search failed for query=%r: %s", query, e)
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
from deerflow.config import get_app_config
try:
from tavily import TavilyClient
except ImportError:
logger.warning("tavily not installed, falling back to demo search")
return []
config = get_app_config().get_tool_config("web_search")
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
if not api_key:
logger.warning("Tavily API key not configured, falling back to demo search")
return []
client = TavilyClient(api_key=api_key)
res = client.search(query, max_results=max_results)
results = []
for item in res.get("results", []):
content = item.get("content", "")
results.append(SearchResult(
title=item.get("title", ""),
content=content[:500],
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
published_at=item.get("published_date", ""),
url=item.get("url", ""),
relevance_score=item.get("score", 0.0),
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,114 @@
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
from __future__ import annotations
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
_DEFAULT_KB = "samples"
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
_RAW_SNIPPET_LIMIT = 2000
class IntranetSearchProvider(BaseSearchProvider):
"""内网知识库搜索接口适配器。
接口:POST {endpoint}
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
"""
def __init__(
self,
base_url: str = "",
knowledge_base_name: str = _DEFAULT_KB,
timeout: int = 120,
**_ignored,
):
self.endpoint = base_url or _DEFAULT_ENDPOINT
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
# 内网 ES 接口可能较慢,超时放宽到 2 分钟
self.timeout = timeout
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
self.errors: list[str] = []
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
self.raw_responses: list[str] = []
def _record_raw(self, body: str) -> None:
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
body = (body or "").strip()
if not body:
return
if len(body) > _RAW_SNIPPET_LIMIT:
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
self.raw_responses.append(body)
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
import httpx
payload = {
"query": query,
"knowledge_base_name": self.knowledge_base_name,
"ts": "yes",
}
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
raw_text = ""
try:
async with httpx.AsyncClient(timeout=self.timeout) as client:
resp = await client.post(self.endpoint, json=payload)
resp.raise_for_status()
raw_text = resp.text
data = resp.json()
except httpx.HTTPStatusError as e:
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
self._record_raw(e.response.text if e.response is not None else "")
return []
except Exception as e:
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
self._record_raw(raw_text)
return []
# 正常情况下接口直接返回数组;兼容个别情况下包了一层 data/results 的结构
if isinstance(data, list):
items = data
elif isinstance(data, dict):
items = data.get("data") or data.get("results") or []
else:
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
self._record_raw(raw_text)
return []
results: List[SearchResult] = []
for i, item in enumerate(items[:max_results]):
md = (item or {}).get("metadata") or {}
content = md.get("content") or (item or {}).get("page_content") or ""
results.append(SearchResult(
title=str(md.get("m_title") or "").strip() or "(无标题)",
content=str(content)[:500],
source=str(md.get("source1") or "").strip() or "内网知识库",
published_at=str(md.get("m_publish") or "").strip(),
url="", # 内网素材的原文地址由前端依据 recUuid 拼接
# 接口未返回相关度分值,按 ES 返回顺序赋递减分以保留排序
relevance_score=max(0.5, 0.95 - i * 0.03),
source_type="intranet",
recUuid=str(md.get("recUuid") or "").strip(),
))
return results
async def health_check(self) -> bool:
try:
await self.search("测试", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,108 @@
from typing import Annotated, List, Literal, Optional, TypedDict
import operator
# ── 素材 ──────────────────────────────────────────────────────────────────────
class Material(TypedDict):
id: str
title: str
content: str # 截断到 500 字
source: str
published_at: str
relevance_score: float
source_type: str # "internet" | "intranet"
url: str
recUuid: str
class MaterialPackage(TypedDict):
keywords: List[str]
materials: List[Material]
summary: str
# ── 大纲 ──────────────────────────────────────────────────────────────────────
class OutlineSection(TypedDict):
section_title: str
key_points: List[str]
material_ids: List[str]
class Outline(TypedDict):
title: str
sections: List[OutlineSection]
# ── 草稿 ──────────────────────────────────────────────────────────────────────
class DraftArticle(TypedDict):
title: str
outline: Outline
full_markdown: str
word_count: int
revision_count: int
# ── 编辑审核 ──────────────────────────────────────────────────────────────────
class ReviewIssue(TypedDict):
category: str # "fact" | "logic" | "language"
severity: str # "major" | "minor"
location: str
issue: str
suggestion: str
class ReviewResult(TypedDict):
verdict: str # "pass" | "reject"
fact_score: int
logic_score: int
language_score: int
overall_score: int
pass_threshold: int
revision_notes: List[ReviewIssue]
# ── 用户干预 ──────────────────────────────────────────────────────────────────
class UserInterventionPayload(TypedDict):
action: str
approved_material_ids: List[str]
extra_keywords: List[str]
edited_outline: Optional[Outline]
outline_feedback: str
user_revision_notes: str
override_verdict: bool
# ── 主状态 ────────────────────────────────────────────────────────────────────
WritingStatus = Literal[
"researching", "awaiting_material_confirm",
"writing_outline", "awaiting_outline_confirm",
"writing_draft", "awaiting_draft_confirm",
"reviewing", "awaiting_review_confirm",
"revising", "done", "error",
]
class AIWritingState(TypedDict):
# 写作配置
user_intent: str
article_type: str
target_audience: str
word_count_target: int
# Agent A 输出
material_package: Optional[MaterialPackage]
# Agent B 输出
current_outline: Optional[Outline]
drafts: Annotated[List[DraftArticle], operator.add]
current_draft: Optional[DraftArticle]
# Agent C 输出
review_result: Optional[ReviewResult]
# 用户干预数据(每个干预点会覆盖)
last_user_intervention: Optional[UserInterventionPayload]
# 流程控制
revision_count: int
max_revisions: int
status: WritingStatus
# SSE 进度推送队列(每个节点追加)
progress_events: Annotated[List[dict], operator.add]
@@ -5,7 +5,7 @@ import logging
from langchain.agents import create_agent
from langchain_core.runnables import RunnableConfig
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
from deerflow.agents.canvas_agent.prompt import build_canvas_system_prompt
from deerflow.agents.canvas_agent.tools import update_artifact_tool
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
@@ -49,7 +49,19 @@ def make_canvas_agent(config: RunnableConfig):
if thinking_enabled and model_config and not model_config.supports_thinking:
thinking_enabled = False
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
canvas_artifact_title = cfg.get("canvas_artifact_title") or None
canvas_artifact_type = cfg.get("canvas_artifact_type") or None
system_prompt = build_canvas_system_prompt(
artifact_title=canvas_artifact_title,
artifact_type=canvas_artifact_type,
)
logger.info(
"Create CanvasAgent -> model_name: %s, thinking_enabled: %s, artifact_title: %s",
model_name,
thinking_enabled,
canvas_artifact_title,
)
middlewares = [
DanglingToolCallMiddleware(),
@@ -63,6 +75,6 @@ def make_canvas_agent(config: RunnableConfig):
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
tools=[update_artifact_tool, ask_clarification_tool],
middleware=middlewares,
system_prompt=CANVAS_SYSTEM_PROMPT,
system_prompt=system_prompt,
state_schema=ThreadState,
)
@@ -1,4 +1,18 @@
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
def build_canvas_system_prompt(
artifact_title: str | None = None,
artifact_type: str | None = None,
) -> str:
existing_artifact_block = ""
if artifact_title:
type_label = "代码" if artifact_type == "code" else "文档"
existing_artifact_block = f"""
<existing_artifact>
用户当前有一个{type_label}制品,标题为 "{artifact_title}"。
请继续编辑该制品——除非用户明确要求新建,否则不要创建新制品。
</existing_artifact>
"""
return f"""You are a Canvas AI assistant that helps users create and edit documents and code.
<canvas_behavior>
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
@@ -10,11 +24,53 @@ Rules:
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
</canvas_behavior>
<clarification_system>
Before creating a brand-new artifact or performing a major rewrite, verify you have enough
information to do the job well. When key details are missing, call `ask_clarification` FIRST
— do NOT start generating the artifact and ask questions mid-way.
**MUST clarify when (no existing artifact or major rewrite requested):**
1. OUTPUT TYPE IS VAGUE
- "帮我做个主页" / "make a homepage" → ask: tech stack (plain HTML / React / Vue?),
design style, which sections to include
- "写个报告" / "write a report" → ask: topic, target audience, expected length,
required sections
- "写个脚本" / "write a script" → ask: programming language, what it should do
2. REQUIRED CONTENT IS MISSING
- User wants a personal profile page but hasn't provided their name / bio / links
- User wants a data table but hasn't provided the data or schema
3. AMBIGUOUS APPROACH WITH SIGNIFICANTLY DIFFERENT OUTCOMES
- "Design a chat system" → ask: client–server or peer-to-peer, persistence needed?
- "Write an intro" → ask: formal essay or casual blog post?
**DO NOT clarify for:**
- Simple edits to an existing artifact: "make it bold", "fix the typo", "shorten paragraph 2"
- Formatting / style tweaks: "add comments", "use camelCase", "format the code"
- Quick actions: expand / shorten / formalize / casualize / add_comments / fix_bugs / optimize_code
- Requests where intent is clear from context (e.g., user pastes content and says "translate this")
- Follow-up turns where enough context already exists in the conversation
**Format for clarification (use options when practical):**
ask_clarification(
question="你希望这个主页使用什么技术栈?",
clarification_type="missing_info",
options=[
{{"id": "html", "label": "纯 HTML / CSS / JS", "description": "无框架,最简单"}},
{{"id": "react", "label": "React", "description": "组件化,适合复杂交互"}},
{{"id": "vue", "label": "Vue", "description": "渐进式框架,上手容易"}},
],
allow_custom=True,
)
CRITICAL: clarification ALWAYS comes before generating the artifact. Never produce a partial
artifact and then ask for clarification in the same response.
</clarification_system>
<artifact_context>
The current artifact state is injected into your context. When editing, build on the existing content exactly.
When creating from scratch, produce a complete, high-quality artifact in one shot.
</artifact_context>
{existing_artifact_block}
<selection_rewrite>
When the user's message contains a <selection> block, you MUST:
1. Identify the exact selection in the current artifact content.
@@ -38,3 +94,7 @@ When the user triggers a quick action, apply it to the full artifact unless spec
- optimize_code: Improve performance and readability
</quick_actions>
"""
# Backward-compatible constant for callers that don't pass artifact context.
CANVAS_SYSTEM_PROMPT = build_canvas_system_prompt()
@@ -43,9 +43,27 @@ logger = logging.getLogger(__name__)
DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest"
DEFAULT_PORT = 8080
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
PACKAGE_MANAGER_ENV_KEYS = (
"PIP_INDEX_URL",
"PIP_EXTRA_INDEX_URL",
"PIP_TRUSTED_HOST",
"PIP_CERT",
"PIP_CLIENT_CERT",
"UV_INDEX_URL",
"UV_EXTRA_INDEX_URL",
"UV_DEFAULT_INDEX",
"UV_INDEX",
"UV_TRUSTED_HOST",
"HTTP_PROXY",
"HTTPS_PROXY",
"NO_PROXY",
"http_proxy",
"https_proxy",
"no_proxy",
)
def _lock_file_exclusive(lock_file) -> None:
@@ -142,9 +160,9 @@ class AioSandboxProvider(SandboxProvider):
Local provider manages container lifecycle directly (start/stop).
"""
provisioner_url = self._config.get("provisioner_url")
if provisioner_url:
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
return RemoteSandboxBackend(provisioner_url=provisioner_url)
if provisioner_url:
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
return RemoteSandboxBackend(provisioner_url=provisioner_url, environment=self._config["environment"])
logger.info("Using local container sandbox backend")
return LocalContainerBackend(
@@ -172,20 +190,24 @@ class AioSandboxProvider(SandboxProvider):
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
"mounts": sandbox_config.mounts or [],
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
"provisioner_url": getattr(sandbox_config, "provisioner_url", None) or "",
}
@staticmethod
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
"""Resolve environment variable references (values starting with $)."""
resolved = {}
for key, value in env_config.items():
if isinstance(value, str) and value.startswith("$"):
env_name = value[1:]
resolved[key] = os.environ.get(env_name, "")
else:
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
"""Resolve environment variable references (values starting with $)."""
resolved = {}
for key in PACKAGE_MANAGER_ENV_KEYS:
value = os.environ.get(key)
if value:
resolved[key] = value
for key, value in env_config.items():
if isinstance(value, str) and value.startswith("$"):
env_name = value[1:]
resolved[key] = os.environ.get(env_name, "")
else:
resolved[key] = str(value)
return resolved
@@ -40,14 +40,17 @@ class RemoteSandboxBackend(SandboxBackend):
provisioner_url: http://provisioner:8002
"""
def __init__(self, provisioner_url: str):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
"""
self._provisioner_url = provisioner_url.rstrip("/")
def __init__(self, provisioner_url: str, environment: dict[str, str] | None = None):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
environment: Environment variables for sandbox pods. The remote
provisioner may ignore this field if it does not support it.
"""
self._provisioner_url = provisioner_url.rstrip("/")
self._environment = environment or {}
@property
def provisioner_url(self) -> str:
@@ -91,10 +94,11 @@ class RemoteSandboxBackend(SandboxBackend):
try:
resp = requests.post(
f"{self._provisioner_url}/api/sandboxes",
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
},
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
"environment": self._environment,
},
timeout=30,
)
resp.raise_for_status()
@@ -11,8 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
No database is initialized.
@@ -21,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
to reference environment variables from .env:
database:
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
"""
from __future__ import annotations
@@ -37,31 +42,40 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres"] = Field(
default="memory",
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
)
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
echo_sql: bool = Field(
default=False,
description="Echo all SQL statements to log (debug only).",
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres only).",
description="Connection pool size for the app ORM engine (postgres/mysql only).",
)
# -- Derived helpers (not user-configured) --
@@ -94,9 +108,16 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class AgentRow(Base):
@@ -16,9 +17,9 @@ class AgentRow(Base):
# Stored as a plain string so the same id can be used as the filesystem
# directory name under {base_dir}/agents/{id}/.
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,11 @@
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
if session_factory is None:
return None
return AIWritingSessionRepository(session_factory)
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
@@ -0,0 +1,28 @@
"""ORM model for AI writing sessions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class AIWritingSessionRow(Base):
__tablename__ = "ai_writing_sessions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -0,0 +1,162 @@
"""SQLAlchemy-backed AI writing session storage."""
from __future__ import annotations
import json
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_UNSET = object()
class AIWritingSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: AIWritingSessionRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
for k in ("completed_interventions", "review_result", "transcript"):
val = d.get(k)
if isinstance(val, str):
try:
d[k] = json.loads(val)
except Exception:
d[k] = None
return d
async def create(
self,
*,
id: str,
title: str,
user_intent: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
now = datetime.now(UTC)
row = AIWritingSessionRow(
id=id,
user_id=resolved_user_id,
title=title,
user_intent=user_intent,
status="in_progress",
draft_title=None,
draft_markdown=None,
completed_interventions=None,
review_result=None,
transcript=None,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
title=_UNSET,
status=_UNSET,
draft_title=_UNSET,
draft_markdown=_UNSET,
completed_interventions=_UNSET,
review_result=_UNSET,
transcript=_UNSET,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
if title is not _UNSET:
row.title = title
if status is not _UNSET:
row.status = status
if draft_title is not _UNSET:
row.draft_title = draft_title
if draft_markdown is not _UNSET:
row.draft_markdown = draft_markdown
if completed_interventions is not _UNSET:
row.completed_interventions = (
json.dumps(completed_interventions, ensure_ascii=False)
if completed_interventions is not None
else None
)
if review_result is not _UNSET:
row.review_result = (
json.dumps(review_result, ensure_ascii=False)
if review_result is not None
else None
)
if transcript is not _UNSET:
row.transcript = (
json.dumps(transcript, ensure_ascii=False)
if transcript is not None
else None
)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
return self._row_to_dict(row)
async def list(
self,
*,
user_id: str | None | _AutoSentinel = AUTO,
limit: int = 50,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
stmt = select(AIWritingSessionRow)
if resolved_user_id is not None:
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return False
if resolved_user_id is not None and row.user_id != resolved_user_id:
return False
await session.delete(row)
await session.commit()
return True
@@ -0,0 +1,11 @@
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.article_types.sql import ArticleTypeRepository
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
if session_factory is None:
return None
return ArticleTypeRepository(session_factory)
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
@@ -0,0 +1,24 @@
"""ORM model for AI writing article types."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class ArticleTypeRow(Base):
__tablename__ = "ai_writing_article_types"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
label: Mapped[str] = mapped_column(String(128), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -0,0 +1,123 @@
"""SQLAlchemy-backed article type storage."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.article_types.model import ArticleTypeRow
_UNSET = object()
class ArticleTypeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: ArticleTypeRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
return d
async def create(
self,
*,
key: str,
label: str,
description: str | None = None,
prompt_hint: str | None = None,
default_word_count: int = 800,
sort_order: int = 0,
) -> dict:
now = datetime.now(UTC)
row = ArticleTypeRow(
id=str(uuid.uuid4()),
key=key,
label=label,
description=description,
prompt_hint=prompt_hint,
default_word_count=default_word_count,
sort_order=sort_order,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
article_type_id: str,
*,
key=_UNSET,
label=_UNSET,
description=_UNSET,
prompt_hint=_UNSET,
default_word_count=_UNSET,
sort_order=_UNSET,
) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
if key is not _UNSET:
row.key = key
if label is not _UNSET:
row.label = label
if description is not _UNSET:
row.description = description
if prompt_hint is not _UNSET:
row.prompt_hint = prompt_hint
if default_word_count is not _UNSET:
row.default_word_count = default_word_count
if sort_order is not _UNSET:
row.sort_order = sort_order
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(self, article_type_id: str) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
return self._row_to_dict(row)
async def get_by_key(self, key: str) -> dict | None:
async with self._sf() as session:
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return None
return self._row_to_dict(row)
async def list(self) -> list[dict]:
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(self, article_type_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_keys(self) -> list[str]:
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return list(result.scalars())
@@ -10,10 +10,13 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import json
import logging
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
import json
import logging
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
def _json_serializer(obj: object) -> str:
@@ -23,11 +26,37 @@ def _json_serializer(obj: object) -> str:
logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
async def _auto_create_postgres_db(url: str) -> None:
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _add_runtime_vendor_path() -> None:
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
The offline Docker image keeps the Python environment inside the image, while
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
unchanged startup script can still import it after container recreation.
"""
try:
from deerflow.config.runtime_paths import runtime_home
vendor_dir = runtime_home() / "python-vendor"
except Exception:
vendor_dir = Path(".deer-flow/python-vendor").resolve()
vendor = str(vendor_dir)
if vendor_dir.exists() and vendor not in sys.path:
sys.path.insert(0, vendor)
def _mysql_driver_name(url: str) -> str:
if url.startswith("mysql+aiomysql://"):
return "aiomysql"
return "asyncmy"
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
The target database name is extracted from *url*. The connection is
@@ -65,8 +94,8 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", or "postgres".
url: SQLAlchemy async URL (for sqlite/postgres).
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
echo: Echo SQL to log.
pool_size: Postgres connection pool size.
sqlite_dir: Directory to create for SQLite (ensured to exist).
@@ -77,11 +106,20 @@ async def init_engine(
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
if find_spec(driver) is None:
raise ImportError(
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if backend == "sqlite":
import os
@@ -112,16 +150,25 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
@@ -162,10 +209,10 @@ async def init_engine_from_config(config) -> None:
await init_engine("memory")
return
await init_engine(
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, Text, UniqueConstraint
from sqlalchemy import String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class FeedbackRow(Base):
@@ -29,4 +30,4 @@ class FeedbackRow(Base):
comment: Mapped[str | None] = mapped_column(Text)
# Optional text feedback from the user
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Float, Integer, String, Text
from sqlalchemy import Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmCallMetricRow(Base):
@@ -18,7 +19,7 @@ class LlmCallMetricRow(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
# sort/filter by recency.
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
@@ -44,12 +44,13 @@ def run_migrations_offline() -> None:
context.run_migrations()
def do_run_migrations(connection):
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True, # Required for SQLite ALTER TABLE support
)
def do_run_migrations(connection):
render_as_batch = connection.dialect.name == "sqlite"
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
)
with context.begin_transaction():
context.run_migrations()
@@ -7,9 +7,11 @@ Create Date: 2026-05-12
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
from deerflow.persistence.types import PortableJSON
revision = "20260512_01"
@@ -23,9 +25,9 @@ def _has_column(table_name: str, column_name: str) -> bool:
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "result_json"):
op.add_column("scheduled_task_runs", sa.Column("result_json", sa.JSON(), nullable=True))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "result_json"):
op.add_column("scheduled_task_runs", sa.Column("result_json", PortableJSON(), nullable=True))
def downgrade() -> None:
@@ -30,7 +30,7 @@ def upgrade() -> None:
op.create_table(
"agents",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("name", sa.String(191), nullable=False),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("published", sa.Boolean(), nullable=False, server_default=sa.false()),
@@ -28,7 +28,7 @@ def upgrade() -> None:
op.create_table(
"tags",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("name", sa.String(159), nullable=False),
sa.Column("tag_type", sa.String(32), nullable=False),
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
@@ -0,0 +1,50 @@
"""Create ai_writing_sessions table.
Revision ID: 20260517_01
Revises: 20260515_01
Create Date: 2026-05-17
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_01"
down_revision = "20260515_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("ai_writing_sessions"):
return
op.create_table(
"ai_writing_sessions",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("title", sa.String(512), nullable=False, server_default=""),
sa.Column("user_intent", sa.String(512), nullable=True),
sa.Column("status", sa.String(32), nullable=False, server_default="in_progress"),
sa.Column("draft_title", sa.String(512), nullable=True),
sa.Column("draft_markdown", sa.Text, nullable=True),
sa.Column("completed_interventions", sa.Text, nullable=True),
sa.Column("review_result", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_ai_writing_sessions_user_id", "ai_writing_sessions", ["user_id"])
op.create_index("ix_ai_writing_sessions_updated_at", "ai_writing_sessions", ["updated_at"])
def downgrade() -> None:
if _has_table("ai_writing_sessions"):
op.drop_table("ai_writing_sessions")
@@ -0,0 +1,117 @@
"""Create ai_writing_article_types table with default types.
Revision ID: 20260517_02
Revises: 20260517_01
Create Date: 2026-05-17
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_02"
down_revision = "20260517_01"
branch_labels = None
depends_on = None
_DEFAULT_TYPES = [
{
"key": "news",
"label": "新闻资讯",
"description": "时效性强的新闻报道,注重事实准确、简洁客观",
"prompt_hint": "以新闻报道的风格写作,注重5W1H要素,语言简洁客观,突出时效性和事实准确性",
"default_word_count": 600,
"sort_order": 0,
},
{
"key": "analysis",
"label": "分析报道",
"description": "深入分析事件背景与影响,提供专业见解",
"prompt_hint": "以分析报道的风格写作,提供深度背景分析、多角度解读和专业见解,逻辑清晰",
"default_word_count": 1200,
"sort_order": 1,
},
{
"key": "feature",
"label": "深度特稿",
"description": "人物或事件的深度叙事,注重细节与故事性",
"prompt_hint": "以深度特稿的风格写作,注重叙事性和细节描写,人物刻画生动,故事性强,情感真实",
"default_word_count": 2000,
"sort_order": 2,
},
{
"key": "report",
"label": "调研报告",
"description": "数据驱动的专题研究报告,结构严谨",
"prompt_hint": "以调研报告的风格写作,数据详实,结构严谨(摘要/背景/发现/结论),语言专业规范",
"default_word_count": 3000,
"sort_order": 3,
},
]
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("ai_writing_article_types"):
op.create_table(
"ai_writing_article_types",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("key", sa.String(64), nullable=False, unique=True),
sa.Column("label", sa.String(128), nullable=False),
sa.Column("description", sa.Text, nullable=True),
sa.Column("prompt_hint", sa.Text, nullable=True),
sa.Column("default_word_count", sa.Integer, nullable=False, server_default="800"),
sa.Column("sort_order", sa.Integer, nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
# 预置默认文章类型(幂等,跳过已存在的 key)。
# 用 SQLAlchemy Core 构造而非裸 SQL:``key`` 是 MySQL 保留字,裸
# ``SELECT key`` 会触发 1064 语法错误。Core 编译器会按方言自动给保留字
# 加引号(MySQL 反引号 / Postgres 双引号),保持跨后端可移植。
article_types = sa.table(
"ai_writing_article_types",
sa.column("id", sa.String),
sa.column("key", sa.String),
sa.column("label", sa.String),
sa.column("description", sa.Text),
sa.column("prompt_hint", sa.Text),
sa.column("default_word_count", sa.Integer),
sa.column("sort_order", sa.Integer),
sa.column("created_at", sa.DateTime),
sa.column("updated_at", sa.DateTime),
)
conn = op.get_bind()
existing_keys = {row[0] for row in conn.execute(sa.select(article_types.c.key)).fetchall()}
now = datetime.now(UTC)
for t in _DEFAULT_TYPES:
if t["key"] not in existing_keys:
conn.execute(
article_types.insert().values(
id=str(uuid.uuid4()),
key=t["key"],
label=t["label"],
description=t["description"],
prompt_hint=t["prompt_hint"],
default_word_count=t["default_word_count"],
sort_order=t["sort_order"],
created_at=now,
updated_at=now,
)
)
def downgrade() -> None:
if _has_table("ai_writing_article_types"):
op.drop_table("ai_writing_article_types")
@@ -0,0 +1,39 @@
"""Add transcript column to ai_writing_sessions.
存储前端组装的完整对话时间线(progressEvents + completedInterventions),
供历史会话回显出与实时对话一致的富时间线。
Revision ID: 20260517_03
Revises: 20260517_02
Create Date: 2026-05-17
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_03"
down_revision = "20260517_02"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_column("ai_writing_sessions", "transcript"):
return
op.add_column("ai_writing_sessions", sa.Column("transcript", sa.Text, nullable=True))
def downgrade() -> None:
if _has_column("ai_writing_sessions", "transcript"):
op.drop_column("ai_writing_sessions", "transcript")
@@ -0,0 +1,53 @@
"""Create thread_shares table (per-conversation share links).
Revision ID: 20260518_02
Revises: 20260518_01
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_02"
down_revision = "20260518_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
op.create_table(
"thread_shares",
sa.Column("share_code", sa.String(32), primary_key=True),
sa.Column("owner_user_id", sa.String(64), nullable=False),
sa.Column("thread_id", sa.String(64), nullable=False),
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
elif not _has_column("thread_shares", "thread_id"):
# An earlier all-conversations variant of this table — add the column.
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
if _has_table("thread_shares"):
op.drop_table("thread_shares")
@@ -0,0 +1,57 @@
"""Ensure thread_shares.thread_id exists (per-conversation share links).
Standalone migration so databases that already applied an earlier
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
column) get the column added. Idempotent — a no-op when the column is
already present.
Revision ID: 20260518_03
Revises: 20260518_02
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_03"
down_revision = "20260518_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
# Fresh database — 20260518_02 already created the table with thread_id.
return
if not _has_column("thread_shares", "thread_id"):
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
# Leave the column in place — dropping it would break the share feature.
pass
@@ -0,0 +1,39 @@
"""Add retry_count to scheduled_task_runs.
Tracks how many recovery attempts a run has consumed so the scheduler can
cap retries on stuck/timed-out runs.
Revision ID: 20260518_04
Revises: 20260518_03
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_04"
down_revision = "20260518_03"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "retry_count"):
op.add_column(
"scheduled_task_runs",
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
if _has_column("scheduled_task_runs", "retry_count"):
op.drop_column("scheduled_task_runs", "retry_count")
@@ -0,0 +1,44 @@
"""Add mode to thread_shares.
Distinguishes "import" shares (recipient copies the conversation into their
account) from "view" shares (a public, read-only static page).
Revision ID: 20260518_05
Revises: 20260518_04
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_05"
down_revision = "20260518_04"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return True # table missing entirely — skip (created with the column)
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("thread_shares", "mode"):
op.add_column(
"thread_shares",
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if "thread_shares" in inspector.get_table_names() and any(
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
):
op.drop_column("thread_shares", "mode")
@@ -0,0 +1,28 @@
"""Merge the ai-writing and thread-shares migration heads.
Two independent migration branches diverged from ``20260515_01``: the AI
writing feature (``20260517_01..03``) and the thread-shares / scheduled-task
work (``20260518_01..05``). Merging the two branches into one repo left Alembic
with multiple heads, so ``upgrade head`` failed. This is a no-op merge revision
that joins both lineages — it changes no schema.
Revision ID: 20260518_06
Revises: 20260517_03, 20260518_05
Create Date: 2026-05-18
"""
from __future__ import annotations
revision = "20260518_06"
down_revision = ("20260517_03", "20260518_05")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -32,6 +32,7 @@ from deerflow.persistence.scheduled_tasks.model import (
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.user.model import UserRow
__all__ = [
@@ -51,5 +52,6 @@ __all__ = [
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"UserRow",
]
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
from sqlalchemy import Index, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class RunEventRow(Base):
@@ -24,9 +25,9 @@ class RunEventRow(Base):
category: Mapped[str] = mapped_column(String(16), nullable=False)
# "message" | "trace" | "lifecycle"
content: Mapped[str] = mapped_column(Text, default="")
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
event_metadata: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
seq: Mapped[int] = mapped_column(nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
__table_args__ = (
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
@@ -5,10 +5,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import JSON, Boolean, DateTime, String, Text
from sqlalchemy import Boolean, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class NotificationRow(Base):
@@ -19,6 +20,6 @@ class NotificationRow(Base):
type: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
payload: Mapped[dict[str, Any]] = mapped_column(PortableJSON(), nullable=False, default=dict)
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True)
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class RecommendedQuestionRow(Base):
@@ -17,5 +18,5 @@ class RecommendedQuestionRow(Base):
title: Mapped[str] = mapped_column(String(256), nullable=False)
content: Mapped[str] = mapped_column(Text, nullable=False)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, Index, String, Text
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class RunRow(Base):
@@ -22,8 +23,8 @@ class RunRow(Base):
model_name: Mapped[str | None] = mapped_column(String(128))
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
kwargs_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
error: Mapped[str | None] = mapped_column(Text)
# Convenience fields (for listing pages without querying RunEventStore)
@@ -43,7 +44,7 @@ class RunRow(Base):
# Follow-up association
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
raise NotImplementedError
# --- Admin (cross-user) ---
@abstractmethod
async def list_all_tasks(self) -> list[dict[str, Any]]:
"""Return every task across all users. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def delete_task_any(self, task_id: str) -> bool:
"""Delete a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs for a task without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs still in a non-terminal state whose ``started_at`` is older
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
or were orphaned by a service restart."""
raise NotImplementedError
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@@ -56,6 +85,11 @@ class ScheduledTaskStore(ABC):
async def delete_run(self, run_id: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def delete_run_any(self, run_id: str) -> bool:
"""Delete a run by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
raise NotImplementedError
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
due.sort(key=lambda task: task["next_run_at"])
return due[:limit]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
return [dict(task) for task in self._tasks.values()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
if task is None:
return None
task.update(data)
return dict(task)
async def delete_task_any(self, task_id: str) -> bool:
return self._tasks.pop(task_id, None) is not None
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
return runs[:limit]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
stuck = [
dict(run)
for run in self._runs.values()
if run.get("status") in ("running", "pending")
and run.get("started_at") is not None
and run["started_at"] <= cutoff
]
stuck.sort(key=lambda run: run["started_at"])
return stuck[:limit]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
key = (data["task_id"], data["scheduled_for"].isoformat())
if key in self._runs_by_key:
@@ -90,11 +121,21 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
run = self._runs.get(run_id)
if run is None or run["user_id"] != user_id:
return False
self._runs.pop(run_id, None)
self._drop_run(run)
return True
async def delete_run_any(self, run_id: str) -> bool:
run = self._runs.get(run_id)
if run is None:
return False
self._drop_run(run)
return True
def _drop_run(self, run: dict[str, Any]) -> None:
self._runs.pop(run["id"], None)
scheduled_for = run.get("scheduled_for")
if hasattr(scheduled_for, "isoformat"):
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class SchedulerThreadRow(Base):
@@ -15,8 +16,8 @@ class SchedulerThreadRow(Base):
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRow(Base):
@@ -29,20 +30,20 @@ class ScheduledTaskRow(Base):
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
name: Mapped[str] = mapped_column(String(256))
name: Mapped[str] = mapped_column(String(127))
prompt: Mapped[str] = mapped_column(Text)
schedule_text: Mapped[str] = mapped_column(String(256))
cron_expr: Mapped[str] = mapped_column(String(128))
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), index=True)
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
last_status: Mapped[str | None] = mapped_column(String(32))
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
execution_context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
published: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskSubscriptionRow(Base):
@@ -53,8 +54,8 @@ class ScheduledTaskSubscriptionRow(Base):
task_id: Mapped[str] = mapped_column(String(64), index=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
notify_element: Mapped[bool] = mapped_column(Boolean, default=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRunRow(Base):
@@ -66,21 +67,25 @@ class ScheduledTaskRunRow(Base):
user_id: Mapped[str] = mapped_column(String(64), index=True)
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), index=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
status: Mapped[str] = mapped_column(String(32), default="pending")
error: Mapped[str | None] = mapped_column(Text)
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
result_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
# Number of recovery attempts made after the first try. A run is retried
# when it overruns the per-attempt timeout or is found orphaned in the
# "running" state (e.g. after a service restart). See ScheduledTaskService.
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskDeliveryProfileRow(Base):
__tablename__ = "scheduled_task_delivery_profiles"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
element_user_id: Mapped[str | None] = mapped_column(String(191), index=True)
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
)
return [_to_dict(row) for row in result.scalars()]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
return [_to_dict(row) for row in result.scalars()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return None
for key, value in data.items():
attr = "execution_context_json" if key == "execution_context" else key
if hasattr(row, attr):
setattr(row, attr, value)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def delete_task_any(self, task_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.task_id == task_id)
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(
ScheduledTaskRunRow.status.in_(("running", "pending")),
ScheduledTaskRunRow.started_at.is_not(None),
ScheduledTaskRunRow.started_at <= cutoff,
)
.order_by(ScheduledTaskRunRow.started_at.asc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
row = ScheduledTaskRunRow(
id=data["id"],
@@ -180,6 +234,15 @@ class ScheduledTaskRepository(ScheduledTaskStore):
await session.commit()
return True
async def delete_run_any(self, run_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRunRow, run_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
@@ -232,7 +295,14 @@ class ScheduledTaskRepository(ScheduledTaskStore):
result = await session.execute(
select(ScheduledTaskRow)
.where(ScheduledTaskRow.published.is_(True))
.order_by(ScheduledTaskRow.published_at.desc().nullslast(), ScheduledTaskRow.created_at.desc())
# ``NULLS LAST`` is unsupported by MySQL — sort by an
# ``IS NULL`` flag instead (False<True ⇒ non-null rows first),
# which is portable across mysql/sqlite/postgres.
.order_by(
ScheduledTaskRow.published_at.is_(None),
ScheduledTaskRow.published_at.desc(),
ScheduledTaskRow.created_at.desc(),
)
.offset(max(0, offset))
.limit(max(1, limit))
)
@@ -14,10 +14,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, UniqueConstraint
from sqlalchemy import String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
# Allowed tag/target types. Kept here so persistence + API share one source.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
@@ -28,12 +29,12 @@ class TagRow(Base):
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
name: Mapped[str] = mapped_column(String(159), nullable=False)
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class TagAssignmentRow(Base):
@@ -44,4 +45,4 @@ class TagAssignmentRow(Base):
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, String
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class ThreadMetaRow(Base):
@@ -18,6 +19,6 @@ class ThreadMetaRow(Base):
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
display_name: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(20), default="idle")
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,19 @@
"""Persistence layer for conversation-history share links."""
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
__all__ = [
"ThreadShareStore",
"MemoryThreadShareStore",
"ThreadShareRepository",
"generate_share_code",
"make_thread_share_store",
]
def make_thread_share_store(session_factory):
if session_factory is None:
return MemoryThreadShareStore()
return ThreadShareRepository(session_factory)
@@ -0,0 +1,50 @@
"""Abstract persistence API for per-conversation share links."""
from __future__ import annotations
import secrets
from abc import ABC, abstractmethod
from typing import Any
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
def generate_share_code(length: int = 8) -> str:
"""Return a random, unguessable, human-readable share code."""
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
class ThreadShareStore(ABC):
@abstractmethod
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
"""Create a new share link for one conversation and return its record.
``mode`` is ``"import"`` (recipient copies the conversation) or
``"view"`` (a public read-only static page).
"""
raise NotImplementedError
@abstractmethod
async def get(self, share_code: str) -> dict[str, Any] | None:
"""Look up a share by code regardless of owner (None if missing)."""
raise NotImplementedError
@abstractmethod
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
Used so re-sharing the same conversation in the same mode reuses its
existing code instead of minting a new one each time.
"""
raise NotImplementedError
@abstractmethod
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
"""List all share links created by ``owner_user_id`` (newest first)."""
raise NotImplementedError
@abstractmethod
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
"""Revoke a share link. Returns False if missing or not owned."""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
class MemoryThreadShareStore(ThreadShareStore):
def __init__(self) -> None:
self._shares: dict[str, dict[str, Any]] = {}
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
code = generate_share_code()
while code in self._shares:
code = generate_share_code()
record = {
"share_code": code,
"owner_user_id": owner_user_id,
"thread_id": thread_id,
"revoked": False,
"mode": mode,
"created_at": datetime.now(UTC).isoformat(),
}
self._shares[code] = record
return copy.deepcopy(record)
async def get(self, share_code: str) -> dict[str, Any] | None:
record = self._shares.get(share_code)
return copy.deepcopy(record) if record is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
matches = [
r
for r in self._shares.values()
if r["owner_user_id"] == owner_user_id
and r["thread_id"] == thread_id
and r.get("mode", "import") == mode
and not r["revoked"]
]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
rows.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in rows]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
record = self._shares.get(share_code)
if record is None or record["owner_user_id"] != owner_user_id:
return False
record["revoked"] = True
return True
@@ -0,0 +1,37 @@
"""ORM model for conversation share links.
A share link is a short, revocable code created by a user for *one* of their
conversations. Redeeming it copies that single conversation into the redeemer's
account. The table only stores the link itself — the conversation is copied
lazily at import time.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ThreadShareRow(Base):
__tablename__ = "thread_shares"
# Short, URL-safe, unguessable code embedded in the share link.
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# The user who created the share.
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# The single conversation (thread) this link shares.
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Soft-disable: a revoked link can no longer be previewed or imported.
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Share mode:
# "import" — recipient (a logged-in user) copies the conversation into
# their own account and can keep chatting in the copy.
# "view" — a public, read-only static page anyone can open without
# logging in; viewers cannot send new messages.
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,80 @@
"""SQLAlchemy-backed per-conversation share-link repository."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.model import ThreadShareRow
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
data = row.to_dict()
created = data.get("created_at")
if isinstance(created, datetime):
data["created_at"] = created.isoformat()
return data
class ThreadShareRepository(ThreadShareStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
# Retry on the (astronomically unlikely) code collision.
async with self._sf() as session:
for _ in range(8):
code = generate_share_code()
if await session.get(ThreadShareRow, code) is not None:
continue
row = ThreadShareRow(
share_code=code,
owner_user_id=owner_user_id,
thread_id=thread_id,
revoked=False,
mode=mode,
)
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
raise RuntimeError("Failed to allocate a unique share code")
async def get(self, share_code: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
return _row_to_dict(row) if row is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
stmt = (
select(ThreadShareRow)
.where(
ThreadShareRow.owner_user_id == owner_user_id,
ThreadShareRow.thread_id == thread_id,
ThreadShareRow.mode == mode,
ThreadShareRow.revoked.is_(False),
)
.order_by(ThreadShareRow.created_at.desc())
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalars().first()
return _row_to_dict(row) if row is not None else None
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
if row is None or row.owner_user_id != owner_user_id:
return False
row.revoked = True
await session.commit()
return True
@@ -0,0 +1,85 @@
"""Portable SQLAlchemy column types."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
from sqlalchemy.types import TypeDecorator
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
class PortableJSON(TypeDecorator):
"""Use native JSON where available, and TEXT on older MySQL servers."""
impl = JSON
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == "mysql":
return dialect.type_descriptor(Text())
return dialect.type_descriptor(JSON())
def process_bind_param(self, value: Any, dialect) -> Any:
if dialect.name != "mysql" or value is None:
return value
return json.dumps(value, ensure_ascii=False)
def process_result_value(self, value: Any, dialect) -> Any:
if dialect.name != "mysql" or value is None or not isinstance(value, str):
return value
if not value:
return None
return json.loads(value)
class BeijingDateTime(TypeDecorator):
"""Store timestamps as Beijing (UTC+8) wall-clock time.
The application works in UTC internally (``datetime.now(UTC)`` everywhere).
MySQL ``DATETIME`` and SQLite have no timezone, so the driver serializes the
wall-clock fields of whatever datetime it is given and drops ``tzinfo``.
Without conversion a row stores the UTC wall clock and an operator browsing
the table sees a time 8 hours behind Beijing.
This decorator converts UTC-aware values to Beijing on the way in (stored as
naive Beijing wall-clock) and re-attaches the Beijing zone on the way out, so
the database holds Beijing time while Python keeps receiving correct
timezone-aware datetimes — instant comparisons stay correct either way.
PostgreSQL has a real ``TIMESTAMP WITH TIME ZONE`` type and stores the
instant unambiguously, so values pass through unchanged there.
"""
impl = DateTime
cache_ok = True
def load_dialect_impl(self, dialect):
# timezone=True keeps PostgreSQL on timestamptz; MySQL/SQLite ignore it.
return dialect.type_descriptor(DateTime(timezone=True))
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
return value.replace(tzinfo=BEIJING_TZ)
return value.astimezone(BEIJING_TZ)
@@ -13,10 +13,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, Index, String, text
from sqlalchemy import Boolean, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class UserRow(Base):
@@ -25,7 +26,7 @@ class UserRow(Base):
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
@@ -33,7 +34,7 @@ class UserRow(Base):
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
@@ -87,6 +87,7 @@ class RunEventStore(abc.ABC):
limit: int = 50,
before_seq: int | None = None,
after_seq: int | None = None,
user_id: str | None = None,
) -> list[dict]:
"""Return displayable messages (category=message) for a specific run, ordered by seq ascending.
@@ -152,7 +152,9 @@ class JsonlRunEventStore(RunEventStore):
events = [e for e in events if e.get("event_type") in event_types]
return events[:limit]
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
# JSONL events carry no per-event owner; user_id is accepted for
# interface parity with DbRunEventStore but ignored.
events = self._read_run_events(thread_id, run_id)
filtered = [e for e in events if e.get("category") == "message"]
if before_seq is not None:
@@ -97,7 +97,9 @@ class MemoryRunEventStore(RunEventStore):
filtered = [e for e in filtered if e["event_type"] in event_types]
return filtered[:limit]
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
# The in-memory store keeps no per-event owner, so user_id is accepted
# for interface parity with DbRunEventStore but ignored.
all_events = self._events.get(thread_id, [])
filtered = [e for e in all_events if e["run_id"] == run_id and e["category"] == "message"]
if before_seq is not None:
@@ -1,4 +1,4 @@
from .registry import get_scheduled_task_service, set_scheduled_task_service
from .service import ScheduledTaskService
from .service import DuplicateTaskNameError, ScheduledTaskService
__all__ = ["ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
__all__ = ["DuplicateTaskNameError", "ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
@@ -9,7 +9,7 @@ import re
import shutil
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -27,6 +27,27 @@ from deerflow.utils.time import now_iso
logger = logging.getLogger(__name__)
# A single execution attempt may not exceed this wall-clock budget. When it
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
# and retried. The watchdog uses the same window to recover runs orphaned by a
# crash or restart — those stay "running" in the DB with no in-process owner.
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
class DuplicateTaskNameError(Exception):
"""A user already owns a scheduled task with the requested name.
Task names are unique per user (``uq_scheduled_tasks_user_name``). The
service raises this instead of letting the raw DB ``IntegrityError`` bubble
up so the API layer can return a friendly 409 rather than a 500.
"""
def __init__(self, name: str) -> None:
self.name = name
super().__init__(f"A scheduled task named {name!r} already exists")
@dataclass
class _RuntimeUser:
@@ -337,15 +358,20 @@ class ScheduledTaskService:
"kind": item.get("kind"),
}
@staticmethod
def _remove_run_archive(run_id: str) -> None:
"""Best-effort cleanup of a run's on-disk result archive."""
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
if archive_dir.exists():
try:
shutil.rmtree(archive_dir)
except Exception:
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
async def delete_run(self, user_id: str, run_id: str) -> bool:
deleted = await self.store.delete_run(run_id, user_id)
if deleted:
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
if archive_dir.exists():
try:
shutil.rmtree(archive_dir)
except Exception:
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
self._remove_run_archive(run_id)
return True
candidate = await self.store.get_run_any(run_id)
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
@@ -358,6 +384,20 @@ class ScheduledTaskService:
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
return await self.store.update_delivery_profile(user_id, data)
async def _assert_name_available(self, user_id: str, name: str, *, exclude_task_id: str | None = None) -> None:
"""Raise :class:`DuplicateTaskNameError` if ``user_id`` already owns ``name``.
Enforces ``uq_scheduled_tasks_user_name`` proactively so callers get a
clean conflict error instead of a raw DB ``IntegrityError``. The check
compares the trimmed name; ``exclude_task_id`` skips the task being
renamed so an unchanged name on update is not flagged as a collision.
"""
for existing in await self.store.list_tasks(user_id):
if exclude_task_id is not None and existing.get("task_id") == exclude_task_id:
continue
if (existing.get("name") or "").strip() == name:
raise DuplicateTaskNameError(name)
async def create_task(
self,
*,
@@ -374,6 +414,8 @@ class ScheduledTaskService:
tz = _normalize_timezone(timezone)
cron_expr = _schedule_to_cron(schedule)
next_run_at = _next_run_at(cron_expr, tz)
final_name = name.strip() or "Scheduled task"
await self._assert_name_available(user_id, final_name)
scheduler_thread = await self.ensure_scheduler_thread(user_id)
context = dict(execution_context or {})
if execution_agent_name:
@@ -386,7 +428,7 @@ class ScheduledTaskService:
"source_thread_id": source_thread_id,
"source_agent_name": source_agent_name,
"execution_agent_name": execution_agent_name,
"name": name.strip() or "Scheduled task",
"name": final_name,
"prompt": prompt.strip(),
"schedule_text": schedule.strip(),
"cron_expr": cron_expr,
@@ -398,31 +440,106 @@ class ScheduledTaskService:
)
return task
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
@staticmethod
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
"""Translate an API update payload into a storage update dict.
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
so both apply identical schedule/timezone normalization.
"""
update_data: dict[str, Any] = {}
for key in ("name", "prompt", "enabled", "execution_agent_name"):
if key in data and data[key] is not None:
update_data[key] = data[key]
# Names are matched against the per-user uniqueness constraint, so keep
# the stored value trimmed to match create_task's normalization.
if isinstance(update_data.get("name"), str):
update_data["name"] = update_data["name"].strip() or "Scheduled task"
timezone = data.get("timezone")
if timezone is not None:
update_data["timezone"] = _normalize_timezone(timezone)
if data.get("schedule") is not None:
schedule = str(data["schedule"])
cron_expr = _schedule_to_cron(schedule)
tz = update_data.get("timezone")
if tz is None:
existing = await self.store.get_task(task_id, user_id)
tz = (existing or {}).get("timezone", "UTC")
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["schedule_text"] = schedule
update_data["cron_expr"] = cron_expr
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
if data.get("execution_context") is not None:
update_data["execution_context"] = data["execution_context"]
# Re-enabling a task must also schedule it forward from now, otherwise a
# paused task with a stale/empty next_run_at would never fire again.
if update_data.get("enabled") is True and "next_run_at" not in update_data:
cron = (existing or {}).get("cron_expr")
if cron:
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["next_run_at"] = _next_run_at(cron, tz)
return update_data
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
existing = await self.store.get_task(task_id, user_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
if "name" in update_data:
await self._assert_name_available(user_id, update_data["name"], exclude_task_id=task_id)
return await self.store.update_task(task_id, user_id, update_data)
async def delete_task(self, user_id: str, task_id: str) -> bool:
return await self.store.delete_task(task_id, user_id)
# --- Admin (cross-user management) ---
async def admin_list_tasks(self) -> list[dict[str, Any]]:
"""List every scheduled task across all users. Admin-only callers."""
tasks = await self.store.list_all_tasks()
for task in tasks:
task["role"] = "owner"
return tasks
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any task regardless of owner. Admin-only callers."""
existing = await self.store.get_task_any(task_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
if "name" in update_data:
await self._assert_name_available(existing["user_id"], update_data["name"], exclude_task_id=task_id)
return await self.store.update_task_any(task_id, update_data)
async def admin_delete_task(self, task_id: str) -> bool:
"""Delete any task regardless of owner. Admin-only callers."""
return await self.store.delete_task_any(task_id)
async def admin_delete_run(self, run_id: str) -> bool:
"""Delete a single run regardless of owner. Admin-only callers."""
deleted = await self.store.delete_run_any(run_id)
if deleted:
self._remove_run_archive(run_id)
return deleted
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""List runs for any task regardless of owner. Admin-only callers."""
runs = await self.store.list_runs_any(task_id, limit=limit)
for run in runs:
run["can_edit"] = True
return runs
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
"""Fetch a single run regardless of owner. Admin-only callers."""
run = await self.store.get_run_any(run_id)
if run is not None:
run["can_edit"] = True
return run
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
"""Trigger an immediate run of any task regardless of owner."""
task = await self.store.get_task_any(task_id)
if task is None:
return None
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
return await self.store.update_task(task_id, user_id, {"enabled": False})
@@ -437,11 +554,19 @@ class ScheduledTaskService:
task = await self.store.get_task(task_id, user_id)
if task is None:
return None
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def _run_loop(self) -> None:
while not self._stopping.is_set():
# Recover hung / orphaned runs first so a single stuck task cannot
# silently wedge the scheduler indefinitely.
try:
await self._reconcile_stuck_runs()
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Scheduled task watchdog scan failed")
try:
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
for task in due:
@@ -466,64 +591,195 @@ class ScheduledTaskService:
finally:
self._running_task_ids.discard(task["task_id"])
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
"""Run a task while marking it busy so the watchdog won't double-drive it."""
self._running_task_ids.add(task["task_id"])
try:
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
finally:
self._running_task_ids.discard(task["task_id"])
async def _reconcile_stuck_runs(self) -> None:
"""Recover runs left in the ``running`` state with no in-process owner.
A run becomes stuck when the agent loop hangs past the per-attempt
timeout, or when the process crashes / restarts mid-run (the original
asyncio task is gone but the DB row still says ``running``). For each
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
``failed`` so it no longer shows as perpetually executing.
"""
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
for run in stuck:
task_id = run["task_id"]
# A run still owned by a live in-process execution is handled by
# that execution's own timeout/retry loop — don't touch it here.
if task_id in self._running_task_ids:
continue
self._running_task_ids.add(task_id)
asyncio.create_task(self._recover_and_untrack(run))
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
try:
await self._recover_stuck_run(run)
except Exception:
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
finally:
self._running_task_ids.discard(run["task_id"])
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
run_id = run["id"]
retry_count = int(run.get("retry_count") or 0)
task = await self.store.get_task_any(run["task_id"])
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
if task is None:
# Parent task was deleted — just close out the orphaned run.
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
)
return
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
async def _fail_run(
self,
task: dict[str, Any],
run_id: str,
scheduled_for: datetime,
*,
error: str,
retry_count: int = 0,
) -> None:
"""Mark a run failed and reflect the outcome on the parent task."""
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
async def _execute_task(
self,
task: dict[str, Any],
*,
scheduled_for: datetime,
manual: bool,
run_row: dict[str, Any] | None = None,
) -> None:
# Each run uses its own throwaway thread so message history cannot accumulate
# across runs and blow past the model's input window. The per-user
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
# we keep it on the task row only for backwards compatibility.
runtime_thread_id = str(uuid.uuid4())
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": runtime_thread_id,
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
is_recovery = run_row is not None
if run_row is None:
return
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": str(uuid.uuid4()),
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
if run_row is None:
return
else:
# Recovery path: re-claim an existing (orphaned) run row.
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
if not manual:
# Advancing the schedule only happens once, when the run is first
# created — recovery retries reuse the original scheduled_for.
if not manual and not is_recovery:
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
try:
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_row["id"],
output_snapshot=output_snapshot,
run_id = run_row["id"]
retry_count = int(run_row.get("retry_count") or 0)
while True:
runtime_thread_id = str(uuid.uuid4())
try:
result = await asyncio.wait_for(
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
)
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
except TimeoutError:
# The attempt overran the 30-minute cap. Retry a bounded number
# of times, then give up and mark the run failed.
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
retry_count += 1
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
await self.store.update_run(
run_id,
{
"retry_count": retry_count,
"status": "running",
"started_at": datetime.now(UTC),
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
},
)
continue
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
return
await self.store.update_run(
run_row["id"],
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
run_id,
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
await self._deliver_element_result(task, result["content"])
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
result = _build_result(
task,
scheduled_for=scheduled_for,
content=f"Scheduled task failed: {exc}",
return
async def _run_agent_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Run the agent once and build the result payload.
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
cannot hold the shared run lock — and the whole scheduler — forever.
"""
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_id,
output_snapshot=output_snapshot,
)
await self.store.update_run(
run_row["id"],
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
token = set_current_user(_RuntimeUser(id=user_id))
@@ -41,13 +41,14 @@ dependencies = [
[project.optional-dependencies]
hindsight = ["hindsight-client>=0.4.22"]
ollama = ["langchain-ollama>=0.3.0"]
postgres = [
"asyncpg>=0.29",
"langgraph-checkpoint-postgres>=3.0.5",
"psycopg[binary]>=3.3.3",
"psycopg-pool>=3.3.0",
]
pymupdf = ["pymupdf4llm>=0.0.17"]
postgres = [
"asyncpg>=0.29",
"langgraph-checkpoint-postgres>=3.0.5",
"psycopg[binary]>=3.3.3",
"psycopg-pool>=3.3.0",
]
mysql = ["asyncmy>=0.2.10"]
pymupdf = ["pymupdf4llm>=0.0.17"]
[build-system]
requires = ["hatchling"]