Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xuhd
This commit is contained in:
commit
070cd07199
372 files changed
+86240
-4969
No files matched your search
+3
@@ -0,0 +1,3 @@
|
||||
from .graph import AI_WRITING_GRAPH, build_ai_writing_graph
|
||||
|
||||
__all__ = ["AI_WRITING_GRAPH", "build_ai_writing_graph"]
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
from langgraph.graph import END, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from .state import AIWritingState
|
||||
from .nodes.intent_parser import intent_parser_node
|
||||
from .nodes.researcher import researcher_node
|
||||
from .nodes.writer_outline import writer_outline_node
|
||||
from .nodes.writer_draft import writer_draft_node
|
||||
from .nodes.editor import editor_node
|
||||
|
||||
# ── 干预节点 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-1:素材确认"""
|
||||
pkg = state["material_package"]
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "material_confirm",
|
||||
"message": "素材收集专家已完成素材搜集,请确认素材后继续。",
|
||||
"materials": pkg["materials"],
|
||||
"keywords": pkg["keywords"],
|
||||
"summary": pkg["summary"],
|
||||
})
|
||||
|
||||
action = user_input.get("action", "confirm")
|
||||
|
||||
if action == "confirm":
|
||||
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
|
||||
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
|
||||
updated_pkg = {**pkg, "materials": filtered}
|
||||
return {
|
||||
"material_package": updated_pkg,
|
||||
"last_user_intervention": user_input,
|
||||
"status": "writing_outline",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材,开始规划大纲"}],
|
||||
}
|
||||
|
||||
# re_search:重新搜索
|
||||
extra_kws = user_input.get("extraKeywords") or []
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
|
||||
"status": "researching",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
|
||||
}
|
||||
|
||||
|
||||
def _normalize_outline(outline: dict) -> dict:
|
||||
"""将前端回传的 camelCase 大纲统一为后端 snake_case 格式。"""
|
||||
if not outline:
|
||||
return outline
|
||||
sections = []
|
||||
for s in outline.get("sections", []):
|
||||
sections.append({
|
||||
"section_title": s.get("section_title") or s.get("sectionTitle", ""),
|
||||
"key_points": s.get("key_points") or s.get("keyPoints", []),
|
||||
"material_ids": s.get("material_ids") or s.get("materialIds", []),
|
||||
})
|
||||
return {"title": outline.get("title", ""), "sections": sections}
|
||||
|
||||
|
||||
async def pause_outline_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-2:大纲确认"""
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "outline_confirm",
|
||||
"message": "作家已完成大纲规划,请确认或编辑大纲后继续。",
|
||||
"outline": state["current_outline"],
|
||||
})
|
||||
|
||||
action = user_input.get("action", "confirm")
|
||||
|
||||
if action == "confirm":
|
||||
raw_outline = user_input.get("editedOutline") or state["current_outline"]
|
||||
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
|
||||
return {
|
||||
"current_outline": edited,
|
||||
"last_user_intervention": user_input,
|
||||
"status": "writing_draft",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}],
|
||||
}
|
||||
|
||||
# re_outline
|
||||
feedback = user_input.get("outlineFeedback", "")
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_feedback": feedback},
|
||||
"status": "writing_outline",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
|
||||
}
|
||||
|
||||
|
||||
async def pause_draft_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-3:草稿确认"""
|
||||
draft = state["current_draft"]
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "draft_confirm",
|
||||
"message": "草稿已完成,请确认或给出修改意见。",
|
||||
"draft_title": draft["title"],
|
||||
"revision_count": draft["revision_count"],
|
||||
"review_result": state.get("review_result"),
|
||||
})
|
||||
|
||||
action = user_input.get("action", "to_editor")
|
||||
|
||||
if action == "finalize":
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户确认定稿"}],
|
||||
}
|
||||
|
||||
if action == "user_revise":
|
||||
notes = user_input.get("userRevisionNotes", "")
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_user_notes": notes},
|
||||
"revision_count": state.get("revision_count", 0) + 1,
|
||||
"status": "revising",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"按用户意见重写({notes[:30]})"}],
|
||||
}
|
||||
|
||||
# to_editor
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "reviewing",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "提交编辑审核"}],
|
||||
}
|
||||
|
||||
|
||||
async def pause_review_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-4:编辑打回确认"""
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "review_confirm",
|
||||
"message": "编辑认为文章需要修改,请决定下一步操作。",
|
||||
"review_result": state["review_result"],
|
||||
"revision_count": state.get("revision_count", 0),
|
||||
"max_revisions": state.get("max_revisions", 3),
|
||||
})
|
||||
|
||||
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户强制定稿"}],
|
||||
}
|
||||
|
||||
action = user_input.get("action", "accept_review")
|
||||
extra_notes = user_input.get("userRevisionNotes", "")
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_extra_notes": extra_notes, "action": action},
|
||||
"revision_count": state.get("revision_count", 0) + 1,
|
||||
"status": "revising",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "按编辑意见重写"}],
|
||||
}
|
||||
|
||||
|
||||
# ── 条件路由 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def after_material_pause(state: AIWritingState) -> str:
|
||||
return "researcher" if state["status"] == "researching" else "writer_outline"
|
||||
|
||||
|
||||
def after_outline_pause(state: AIWritingState) -> str:
|
||||
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
|
||||
|
||||
|
||||
def after_draft_pause(state: AIWritingState) -> str:
|
||||
status = state["status"]
|
||||
if status == "done":
|
||||
return "end"
|
||||
if status == "revising":
|
||||
return "writer_draft"
|
||||
return "editor"
|
||||
|
||||
|
||||
def after_review(state: AIWritingState) -> str:
|
||||
review = state.get("review_result") or {}
|
||||
if review.get("verdict") == "pass":
|
||||
return "pause_draft"
|
||||
rev_count = state.get("revision_count", 0)
|
||||
max_rev = state.get("max_revisions", 3)
|
||||
if rev_count >= max_rev:
|
||||
return "pause_draft" # 超出上限,强制到草稿确认
|
||||
return "pause_review"
|
||||
|
||||
|
||||
def after_review_pause(state: AIWritingState) -> str:
|
||||
return "end" if state["status"] == "done" else "writer_draft"
|
||||
|
||||
|
||||
# ── 组装 Graph ────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_ai_writing_graph():
|
||||
g = StateGraph(AIWritingState)
|
||||
|
||||
g.add_node("intent_parser", intent_parser_node)
|
||||
g.add_node("researcher", researcher_node)
|
||||
g.add_node("pause_material", pause_material_confirm)
|
||||
g.add_node("writer_outline", writer_outline_node)
|
||||
g.add_node("pause_outline", pause_outline_confirm)
|
||||
g.add_node("writer_draft", writer_draft_node)
|
||||
g.add_node("pause_draft", pause_draft_confirm)
|
||||
g.add_node("editor", editor_node)
|
||||
g.add_node("pause_review", pause_review_confirm)
|
||||
|
||||
g.set_entry_point("intent_parser")
|
||||
g.add_edge("intent_parser", "researcher")
|
||||
g.add_edge("researcher", "pause_material")
|
||||
|
||||
g.add_conditional_edges("pause_material", after_material_pause,
|
||||
{"researcher": "researcher", "writer_outline": "writer_outline"})
|
||||
|
||||
g.add_edge("writer_outline", "pause_outline")
|
||||
g.add_conditional_edges("pause_outline", after_outline_pause,
|
||||
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
|
||||
|
||||
g.add_edge("writer_draft", "pause_draft")
|
||||
g.add_conditional_edges("pause_draft", after_draft_pause,
|
||||
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
|
||||
|
||||
g.add_conditional_edges("editor", after_review,
|
||||
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
|
||||
|
||||
g.add_conditional_edges("pause_review", after_review_pause,
|
||||
{"end": END, "writer_draft": "writer_draft"})
|
||||
|
||||
checkpointer = MemorySaver()
|
||||
return g.compile(checkpointer=checkpointer)
|
||||
|
||||
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph()
|
||||
Whitespace-only changes.
+242
@@ -0,0 +1,242 @@
|
||||
"""节点共用工具:JSON 解析容错、LLM 创建。
|
||||
|
||||
弱模型经常输出不规范 JSON(中文标点、结尾多余逗号、被 markdown 包裹、
|
||||
夹带解释文字、嵌套对象)。parse_json 用多级降级策略尽量抢救出结构化结果。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_model(config: RunnableConfig, model_name: str | None = None):
|
||||
"""创建对话模型。
|
||||
|
||||
model_name 显式传入时优先使用(如素材收集专家的检索词/排序专用快模型),
|
||||
否则回退到 configurable.model_name(用户在写作表单里选的主模型)。
|
||||
"""
|
||||
from deerflow.models import create_chat_model
|
||||
name = model_name or config.get("configurable", {}).get("model_name", "") or None
|
||||
return create_chat_model(name=name)
|
||||
|
||||
|
||||
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
|
||||
_FULLWIDTH_FIX = {
|
||||
"“": '"', # 左双引号 “
|
||||
"”": '"', # 右双引号 ”
|
||||
"‘": "'", # 左单引号 ‘
|
||||
"’": "'", # 右单引号 ’
|
||||
":": ":", # 全角冒号 :
|
||||
",": ",", # 全角逗号 ,
|
||||
"(": "(", # 全角左括号 (
|
||||
")": ")", # 全角右括号 )
|
||||
"{": "{", # 全角左花括号 {
|
||||
"}": "}", # 全角右花括号 }
|
||||
"[": "[", # 全角左方括号 [
|
||||
"]": "]", # 全角右方括号 ]
|
||||
"【": "[", # 黑方括号 【
|
||||
"】": "]", # 黑方括号 】
|
||||
}
|
||||
|
||||
|
||||
def _extract_balanced(text: str) -> str | None:
|
||||
"""从文本中提取第一个括号平衡的 {...} 块(支持嵌套,忽略字符串内的括号)。"""
|
||||
start = text.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
depth = 0
|
||||
in_str = False
|
||||
escape = False
|
||||
for i in range(start, len(text)):
|
||||
ch = text[i]
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
continue
|
||||
if ch == '"':
|
||||
in_str = not in_str
|
||||
continue
|
||||
if in_str:
|
||||
continue
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start:i + 1]
|
||||
return None
|
||||
|
||||
|
||||
def _repair_json(text: str) -> str:
|
||||
"""尽力修复弱模型常见的 JSON 错误:全角标点、结尾多余逗号。"""
|
||||
for bad, good in _FULLWIDTH_FIX.items():
|
||||
text = text.replace(bad, good)
|
||||
# 去掉对象/数组结尾多余的逗号: ,} 或 ,]
|
||||
text = re.sub(r",(\s*[}\]])", r"\1", text)
|
||||
return text
|
||||
|
||||
|
||||
def _try_load(text: str) -> dict | None:
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
return result if isinstance(result, dict) else None
|
||||
|
||||
|
||||
def parse_json(text: str, fallback: dict) -> dict:
|
||||
"""从 LLM 响应中提取 JSON 对象,多级降级,失败时返回 fallback。"""
|
||||
if not text or not text.strip():
|
||||
return fallback
|
||||
text = text.strip()
|
||||
|
||||
# 1. 直接解析
|
||||
if (r := _try_load(text)) is not None:
|
||||
return r
|
||||
|
||||
# 2. markdown 代码块内容
|
||||
m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
|
||||
if m:
|
||||
fenced = m.group(1).strip()
|
||||
if (r := _try_load(fenced)) is not None:
|
||||
return r
|
||||
text = fenced # 用代码块内容继续后续步骤
|
||||
|
||||
# 3. 平衡括号提取(处理夹带解释文字、嵌套对象的情况)
|
||||
candidate = _extract_balanced(text)
|
||||
if candidate:
|
||||
if (r := _try_load(candidate)) is not None:
|
||||
return r
|
||||
# 4. 修复后再试
|
||||
if (r := _try_load(_repair_json(candidate))) is not None:
|
||||
return r
|
||||
|
||||
# 5. 整体修复后兜底
|
||||
if (r := _try_load(_repair_json(text))) is not None:
|
||||
return r
|
||||
|
||||
logger.warning("JSON parse failed, using fallback. raw=%r", text[:200])
|
||||
return fallback
|
||||
|
||||
|
||||
def _content_to_text(content) -> str:
|
||||
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(
|
||||
part.get("text", "") if isinstance(part, dict) else str(part)
|
||||
for part in content
|
||||
)
|
||||
return str(content)
|
||||
|
||||
|
||||
# 中文常见标点:参与「字数」统计,与 Word 字数统计 / 用户直觉口径对齐
|
||||
_CJK_PUNCTUATION = ",。、;:?!“”‘’()《》【】「」『』〈〉—…·~"
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
"""统计中文语境下的「字数」。
|
||||
|
||||
口径:汉字逐字计 + 中文标点逐个计 + 连续数字/英文各算 1 字。
|
||||
不计入空白、markdown 语法符号(# * - 等)与 ASCII 标点。
|
||||
前后端共用同一口径(前端 utils/wordCount.ts 实现一致)。
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
cjk = len(re.findall(r"[一-鿿]", text))
|
||||
cjk_punct = sum(1 for c in text if c in _CJK_PUNCTUATION)
|
||||
alnum_runs = len(re.findall(r"[A-Za-z0-9]+", text))
|
||||
return cjk + cjk_punct + alnum_runs
|
||||
|
||||
|
||||
class ThinkTagParser:
|
||||
"""流式 <think>...</think> 标签解析器。
|
||||
|
||||
模型可能在任意 chunk 边界截断标签,通过尾部缓冲处理跨 chunk 边界情况。
|
||||
调用 feed(chunk) 返回 list[tuple[bool, str]],每项为 (is_thinking, text)。
|
||||
"""
|
||||
|
||||
_OPEN_TAG = "<think>"
|
||||
_CLOSE_TAG = "</think>"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._in_think = False
|
||||
self._buf = "" # 潜在标签的部分匹配缓冲
|
||||
|
||||
def feed(self, chunk: str) -> list[tuple[bool, str]]:
|
||||
"""处理一个流式 chunk,返回 (is_thinking, text) 片段列表。"""
|
||||
self._buf += chunk
|
||||
result: list[tuple[bool, str]] = []
|
||||
|
||||
while self._buf:
|
||||
if self._in_think:
|
||||
tag = self._CLOSE_TAG
|
||||
idx = self._buf.find(tag)
|
||||
if idx != -1:
|
||||
if idx > 0:
|
||||
result.append((True, self._buf[:idx]))
|
||||
self._buf = self._buf[idx + len(tag):]
|
||||
self._in_think = False
|
||||
else:
|
||||
partial = self._partial_suffix(self._buf, tag)
|
||||
if partial:
|
||||
result.append((True, self._buf[:-partial]))
|
||||
self._buf = self._buf[-partial:]
|
||||
else:
|
||||
result.append((True, self._buf))
|
||||
self._buf = ""
|
||||
break
|
||||
else:
|
||||
tag = self._OPEN_TAG
|
||||
idx = self._buf.find(tag)
|
||||
if idx != -1:
|
||||
if idx > 0:
|
||||
result.append((False, self._buf[:idx]))
|
||||
self._buf = self._buf[idx + len(tag):]
|
||||
self._in_think = True
|
||||
else:
|
||||
partial = self._partial_suffix(self._buf, tag)
|
||||
if partial:
|
||||
result.append((False, self._buf[:-partial]))
|
||||
self._buf = self._buf[-partial:]
|
||||
else:
|
||||
result.append((False, self._buf))
|
||||
self._buf = ""
|
||||
break
|
||||
|
||||
return [(is_t, t) for (is_t, t) in result if t]
|
||||
|
||||
@staticmethod
|
||||
def _partial_suffix(text: str, tag: str) -> int:
|
||||
"""返回 text 尾部与 tag 前缀匹配的最长长度(>0 说明可能是部分标签)。"""
|
||||
max_check = min(len(tag) - 1, len(text))
|
||||
for length in range(max_check, 0, -1):
|
||||
if tag.startswith(text[-length:]):
|
||||
return length
|
||||
return 0
|
||||
|
||||
|
||||
async def llm_json(system: str, user: str, fallback: dict, config: RunnableConfig, model_name: str | None = None) -> dict:
|
||||
"""调用 LLM,解析 JSON 输出,失败用 fallback。
|
||||
|
||||
model_name 可覆盖默认模型(如检索词/排序使用配置的专用快模型)。
|
||||
"""
|
||||
messages = [SystemMessage(content=system), HumanMessage(content=user)]
|
||||
try:
|
||||
# get_model 放进 try:模型名在 config.yaml 找不到时 create_chat_model 会抛
|
||||
# ValueError,若留在 try 外会逐层上抛使整个图硬崩溃。
|
||||
model = get_model(config, model_name=model_name)
|
||||
resp = await model.ainvoke(messages)
|
||||
return parse_json(_content_to_text(resp.content), fallback)
|
||||
except Exception as e:
|
||||
logger.error("llm_json failed: %s", e, exc_info=True)
|
||||
return {**fallback, "_llm_error": str(e)}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, ReviewResult, ReviewIssue
|
||||
from ..prompts.editor_prompts import (
|
||||
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_PASS_THRESHOLD = 75
|
||||
|
||||
|
||||
def _materials_summary(pkg: dict) -> str:
|
||||
lines = []
|
||||
for m in (pkg.get("materials") or [])[:6]:
|
||||
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:200]}")
|
||||
return "\n".join(lines) or "(无素材)"
|
||||
|
||||
|
||||
def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
|
||||
issues = []
|
||||
for item in (raw_issues or []):
|
||||
issues.append(ReviewIssue(
|
||||
category=category,
|
||||
severity=item.get("severity", "minor"),
|
||||
location=item.get("location", ""),
|
||||
issue=item.get("issue", ""),
|
||||
suggestion=item.get("suggestion", ""),
|
||||
))
|
||||
return issues
|
||||
|
||||
|
||||
async def _stream_one_review(
|
||||
model, user_prompt: str, category: str,
|
||||
sse_queue: asyncio.Queue | None, session_id: str,
|
||||
) -> str:
|
||||
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
|
||||
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=EDITOR_SYSTEM),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("review_chunk", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
return full_text
|
||||
|
||||
|
||||
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
draft = state.get("current_draft")
|
||||
if not draft:
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{"type": "error", "agent_name": "编辑", "message": "草稿为空,无法审核"}],
|
||||
}
|
||||
|
||||
article = draft["full_markdown"]
|
||||
pkg = state.get("material_package") or {}
|
||||
mats = _materials_summary(pkg)
|
||||
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
|
||||
|
||||
REVIEW_TASKS = [
|
||||
("fact", FACT_REVIEW_USER.format(materials_summary=mats, article=article)),
|
||||
("logic", LOGIC_REVIEW_USER.format(article=article)),
|
||||
("language", LANGUAGE_REVIEW_USER.format(article=article)),
|
||||
]
|
||||
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
except Exception as e:
|
||||
logger.error("editor model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error", "agent_name": "编辑",
|
||||
"title": "模型错误", "detail": str(e), "message": "审核失败",
|
||||
}],
|
||||
}
|
||||
results: dict[str, dict] = {}
|
||||
review_errors: list[str] = [] # 分项审查的模型异常,供完成后非致命提示
|
||||
|
||||
_CATEGORY_LABEL = {"fact": "事实性", "logic": "逻辑性", "language": "语言文字"}
|
||||
|
||||
for category, user_prompt in REVIEW_TASKS:
|
||||
fallback = {"score": 80, "issues": []}
|
||||
try:
|
||||
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
|
||||
res = parse_json(full_text, fallback)
|
||||
except Exception as e:
|
||||
logger.warning("editor review [%s] failed: %s", category, e)
|
||||
res = fallback
|
||||
review_errors.append(f"「{_CATEGORY_LABEL.get(category, category)}审查」{type(e).__name__}: {e}")
|
||||
|
||||
results[category] = res
|
||||
|
||||
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
|
||||
if sse_queue is not None:
|
||||
issues = _parse_issues(res.get("issues", []), category)
|
||||
await sse_queue.put(("review_item_done", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"score": int(res.get("score", 80)),
|
||||
"issues": [dict(iss) for iss in issues],
|
||||
}))
|
||||
else:
|
||||
logger.debug("editor [%s] done (no sse_queue)", category)
|
||||
|
||||
# 汇总
|
||||
fact_score = int(results["fact"].get("score", 80))
|
||||
logic_score = int(results["logic"].get("score", 80))
|
||||
lang_score = int(results["language"].get("score", 80))
|
||||
overall_score = (fact_score + logic_score + lang_score) // 3
|
||||
|
||||
all_issues: list[ReviewIssue] = (
|
||||
_parse_issues(results["fact"].get("issues", []), "fact") +
|
||||
_parse_issues(results["logic"].get("issues", []), "logic") +
|
||||
_parse_issues(results["language"].get("issues", []), "language")
|
||||
)
|
||||
all_issues.sort(key=lambda x: (0 if x["severity"] == "major" else 1))
|
||||
|
||||
verdict = "pass" if overall_score >= pass_threshold else "reject"
|
||||
|
||||
review = ReviewResult(
|
||||
verdict=verdict,
|
||||
fact_score=fact_score,
|
||||
logic_score=logic_score,
|
||||
language_score=lang_score,
|
||||
overall_score=overall_score,
|
||||
pass_threshold=pass_threshold,
|
||||
revision_notes=all_issues,
|
||||
)
|
||||
|
||||
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
|
||||
notices: list[dict] = []
|
||||
if review_errors:
|
||||
notices.append({
|
||||
"type": "notice",
|
||||
"agent_name": "编辑",
|
||||
"title": "模型错误",
|
||||
"detail": "部分审查项调用失败,已按默认分兜底。原因:" + ";".join(review_errors[:3]),
|
||||
})
|
||||
|
||||
return {
|
||||
"review_result": review,
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "编辑", "message": "开始审核"},
|
||||
*notices,
|
||||
{
|
||||
"type": "review_ready",
|
||||
"agent_name": "编辑",
|
||||
"message": f"审核完成,综合得分 {overall_score},结论:{'通过' if verdict == 'pass' else '打回修改'}",
|
||||
},
|
||||
],
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState
|
||||
from ._utils import llm_json
|
||||
|
||||
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
|
||||
_DEFAULT_ARTICLE_TYPES: list[dict] = [
|
||||
{"key": "news", "label": "新闻资讯"},
|
||||
{"key": "analysis", "label": "分析报道"},
|
||||
{"key": "feature", "label": "深度特稿"},
|
||||
{"key": "report", "label": "调研报告"},
|
||||
]
|
||||
|
||||
_SYSTEM_TEMPLATE = """你是一名写作助手。
|
||||
你的任务:分析用户的写作意图,提取结构化的写作配置。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- article_type 只能是以下之一:{type_options}。
|
||||
|
||||
【完整示例】
|
||||
输入意图:帮我写一篇面向行业专家的深度文章,分析今年新能源汽车出口为什么大涨,大概两千字
|
||||
正确输出:
|
||||
{{"article_type": "feature", "target_audience": "行业专家", "word_count_target": 2000, "refined_intent": "分析今年新能源汽车出口高速增长的原因"}}"""
|
||||
|
||||
_USER_TEMPLATE = """用户写作意图:{user_intent}
|
||||
|
||||
请分析并输出以下 JSON(如果意图中未明确某字段,给出合理默认值):
|
||||
{{{{
|
||||
"article_type": "{type_options} 之一",
|
||||
"target_audience": "目标读者描述",
|
||||
"word_count_target": 800,
|
||||
"refined_intent": "精炼后的写作意图(一句话)"
|
||||
}}}}"""
|
||||
|
||||
|
||||
def _build_type_options(types: list[dict]) -> str:
|
||||
"""将类型列表格式化为提示词中的枚举说明,如 news(新闻资讯)/ analysis(分析报道)。"""
|
||||
return " / ".join(f'{t["key"]}({t["label"]})' for t in types)
|
||||
|
||||
|
||||
def _coerce_word_count(raw) -> int | None:
|
||||
"""稳健地把模型返回的字数字段转成正整数;无法解析时返回 None。
|
||||
|
||||
弱模型(尤其老模型被陌生词带偏时)常返回「约800字」「两千」「800.0」
|
||||
这类非纯数字形式,直接 int() 会抛 ValueError/TypeError 使整个图崩溃。
|
||||
此处用「提取首个数字串」策略容错,解析不出就返回 None 让调用方保留默认值。
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, bool): # bool 是 int 子类,需先排除
|
||||
return None
|
||||
if isinstance(raw, (int, float)):
|
||||
value = int(raw)
|
||||
return value if value > 0 else None
|
||||
if not isinstance(raw, str):
|
||||
# dict / list 等结构化类型:str() 化后可能误含数字,直接判为无法解析
|
||||
return None
|
||||
m = re.search(r"\d[\d,]*", raw)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
value = int(m.group().replace(",", ""))
|
||||
except ValueError:
|
||||
return None
|
||||
return value if value > 0 else None
|
||||
|
||||
|
||||
async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
# 从 configurable 读取动态类型列表,未配置时退回默认值
|
||||
configurable = (config or {}).get("configurable", {})
|
||||
valid_article_types: list[dict] = configurable.get("valid_article_types") or _DEFAULT_ARTICLE_TYPES
|
||||
valid_keys: list[str] = [t["key"] for t in valid_article_types]
|
||||
default_key: str = valid_keys[0] if valid_keys else "analysis"
|
||||
|
||||
type_options = _build_type_options(valid_article_types)
|
||||
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
|
||||
user = _USER_TEMPLATE.format(
|
||||
user_intent=state["user_intent"],
|
||||
type_options=" | ".join(valid_keys),
|
||||
)
|
||||
|
||||
fallback_type = state.get("article_type", default_key)
|
||||
if fallback_type not in valid_keys:
|
||||
fallback_type = default_key
|
||||
|
||||
result = await llm_json(
|
||||
system,
|
||||
user,
|
||||
fallback={
|
||||
"article_type": fallback_type,
|
||||
"target_audience": state.get("target_audience", "普通读者"),
|
||||
"word_count_target": state.get("word_count_target", 800),
|
||||
"refined_intent": state["user_intent"],
|
||||
},
|
||||
config=config,
|
||||
)
|
||||
|
||||
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
|
||||
# 流程仍可继续,但用户需知道结构化配置可能不准)。
|
||||
progress_events: list[dict] = []
|
||||
if result.get("_llm_error"):
|
||||
progress_events.append({
|
||||
"type": "notice",
|
||||
"agent_name": "系统",
|
||||
"title": "模型错误",
|
||||
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
|
||||
})
|
||||
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
|
||||
|
||||
updates: dict = {"status": "researching", "progress_events": progress_events}
|
||||
|
||||
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
|
||||
raw_type = result.get("article_type", "")
|
||||
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
|
||||
|
||||
if result.get("target_audience"):
|
||||
updates["target_audience"] = result["target_audience"]
|
||||
word_count = _coerce_word_count(result.get("word_count_target"))
|
||||
if word_count is not None:
|
||||
updates["word_count_target"] = word_count
|
||||
if result.get("refined_intent"):
|
||||
updates["user_intent"] = result["refined_intent"]
|
||||
|
||||
return updates
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..search import get_search_provider, SearchResult
|
||||
from ..state import AIWritingState, Material, MaterialPackage
|
||||
from ..prompts.researcher_prompts import (
|
||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
||||
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
|
||||
)
|
||||
from ._utils import llm_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_MATERIALS = 12
|
||||
|
||||
|
||||
def _notice(title: str, detail: str, raw: str = "") -> dict:
|
||||
"""构造一个非致命告警事件,前端在时间线显示「标题 + 真实原因」,不打断写作流程。
|
||||
|
||||
raw 非空时附带接口的原始错误响应体,前端默认折叠展示。
|
||||
"""
|
||||
event = {"type": "notice", "agent_name": "素材收集专家", "title": title, "detail": detail}
|
||||
if raw:
|
||||
event["raw"] = raw
|
||||
return event
|
||||
|
||||
|
||||
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
intent = state["user_intent"]
|
||||
article_type = state.get("article_type", "analysis")
|
||||
target_audience = state.get("target_audience", "普通读者")
|
||||
|
||||
# 检查是否是补充搜索(PAUSE-1 re_search action)
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
|
||||
|
||||
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue = cfg.get("sse_queue")
|
||||
sse_session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
|
||||
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
|
||||
rank_model: str | None = ai_writing_config.get("rank_model") or None
|
||||
|
||||
async def _emit(message: str) -> None:
|
||||
if sse_queue is None:
|
||||
return
|
||||
await sse_queue.put(("research_progress", {
|
||||
"session_id": sse_session_id,
|
||||
"message": message,
|
||||
}))
|
||||
|
||||
# Step 1: 提取关键词
|
||||
await _emit("正在生成检索词…")
|
||||
kw_result = await llm_json(
|
||||
EXTRACT_KEYWORDS_SYSTEM,
|
||||
EXTRACT_KEYWORDS_USER.format(
|
||||
user_intent=intent,
|
||||
article_type=article_type,
|
||||
target_audience=target_audience,
|
||||
),
|
||||
fallback={"keywords": [intent]},
|
||||
config=config,
|
||||
model_name=keyword_model,
|
||||
)
|
||||
keywords: list[str] = kw_result.get("keywords", [intent])
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 2: 并发搜索
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
search_keywords = keywords[:6]
|
||||
total = len(search_keywords)
|
||||
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
|
||||
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
|
||||
completed = 0
|
||||
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
nonlocal completed
|
||||
try:
|
||||
return await provider.search(kw, max_results=3)
|
||||
finally:
|
||||
completed += 1
|
||||
if completed < total:
|
||||
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
else:
|
||||
await _emit(f"已完成全部 {total} 次检索")
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
|
||||
# 合并去重
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials: list[Material] = []
|
||||
existing_pkg = state.get("material_package")
|
||||
if existing_pkg and intervention.get("action") != "re_search":
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
|
||||
search_errors: list[str] = []
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
for r in results:
|
||||
if r.url and r.url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(r.url)
|
||||
raw_materials.append(Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
|
||||
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
|
||||
await _emit("检索完成,正在筛选排序素材…")
|
||||
rank_error = ""
|
||||
if len(raw_materials) > _MAX_MATERIALS:
|
||||
materials_list = "\n".join(
|
||||
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
|
||||
for m in raw_materials
|
||||
)
|
||||
rank_result = await llm_json(
|
||||
RANK_MATERIALS_SYSTEM,
|
||||
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
|
||||
fallback={"ranked_ids": [m["id"] for m in raw_materials[:_MAX_MATERIALS]], "summary": f"共搜索到{len(raw_materials)}条素材"},
|
||||
config=config,
|
||||
model_name=rank_model,
|
||||
)
|
||||
rank_error = rank_result.get("_llm_error", "")
|
||||
ranked_ids: list[str] = rank_result.get("ranked_ids", [m["id"] for m in raw_materials])
|
||||
id_to_mat = {m["id"]: m for m in raw_materials}
|
||||
sorted_materials = [id_to_mat[mid] for mid in ranked_ids if mid in id_to_mat]
|
||||
# 加入未被排序的素材
|
||||
sorted_ids_set = set(ranked_ids)
|
||||
for m in raw_materials:
|
||||
if m["id"] not in sorted_ids_set:
|
||||
sorted_materials.append(m)
|
||||
materials = sorted_materials[:_MAX_MATERIALS]
|
||||
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
|
||||
else:
|
||||
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:_MAX_MATERIALS]
|
||||
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
|
||||
|
||||
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
|
||||
if not materials:
|
||||
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
|
||||
|
||||
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
|
||||
|
||||
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
|
||||
notices: list[dict] = []
|
||||
if kw_result.get("_llm_error"):
|
||||
notices.append(_notice("模型错误", f"检索词生成失败,已用默认关键词兜底。原因:{kw_result['_llm_error']}"))
|
||||
if rank_error:
|
||||
notices.append(_notice("模型错误", f"素材排序失败,已按相关度兜底排序。原因:{rank_error}"))
|
||||
if search_errors:
|
||||
uniq = list(dict.fromkeys(search_errors))
|
||||
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
|
||||
notices.append(_notice(
|
||||
"ES 波动导致检索异常",
|
||||
";".join(uniq[:3]),
|
||||
raw="\n\n----------\n\n".join(raw_dumps[:3]),
|
||||
))
|
||||
|
||||
return {
|
||||
"material_package": pkg,
|
||||
"status": "awaiting_material_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
|
||||
*notices,
|
||||
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
|
||||
],
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, DraftArticle
|
||||
from ..prompts.writer_prompts import DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER
|
||||
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
|
||||
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
|
||||
lines = []
|
||||
for mid in material_ids:
|
||||
if mid in mat_map:
|
||||
m = mat_map[mid]
|
||||
lines.append(f"[{mid}] {m['title']}: {m['content'][:300]}")
|
||||
if not lines:
|
||||
for m in pkg.get("materials", [])[:2]:
|
||||
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:300]}")
|
||||
return "\n".join(lines) if lines else "(无相关素材)"
|
||||
|
||||
|
||||
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
outline = state.get("current_outline")
|
||||
if not outline:
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{"type": "error", "agent_name": "作家", "message": "大纲为空,无法写作"}],
|
||||
}
|
||||
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
revision_count = state.get("revision_count", 0)
|
||||
|
||||
# SSE 流式推送所需
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
# 修改意见(来自用户或编辑)
|
||||
revision_notes = ""
|
||||
if intervention.get("action") == "user_revise":
|
||||
notes = intervention.get("_user_notes") or intervention.get("user_revision_notes", "")
|
||||
if notes:
|
||||
revision_notes = f"\n用户修改意见:{notes}"
|
||||
elif intervention.get("action") in ("accept_review", "custom_revise"):
|
||||
review = state.get("review_result") or {}
|
||||
issues = review.get("revision_notes", [])
|
||||
extra = intervention.get("_extra_notes") or intervention.get("user_revision_notes", "")
|
||||
parts = []
|
||||
for iss in issues[:5]:
|
||||
parts.append(f"- [{iss.get('severity','minor')}] {iss.get('location','')}: {iss.get('suggestion','')}")
|
||||
if extra:
|
||||
parts.append(f"- 用户补充:{extra}")
|
||||
if parts:
|
||||
revision_notes = "\n修改要求:\n" + "\n".join(parts)
|
||||
|
||||
word_count_target = state.get("word_count_target", 800)
|
||||
sections = outline.get("sections", [])
|
||||
n_sections = max(len(sections), 1)
|
||||
# 单节硬下限:总目标 / 节数,乘 1.1 过量系数补偿模型对短目标的天然欠量;单节至少 150 字
|
||||
words_per_section = max(150, int(word_count_target * 1.1 / n_sections))
|
||||
article_title = outline.get("title", "文章")
|
||||
|
||||
# 兼容后端 snake_case 与前端编辑后回传的 camelCase
|
||||
def _section_title(idx: int, sec: dict) -> str:
|
||||
return sec.get("section_title") or sec.get("sectionTitle") or f"章节{idx + 1}"
|
||||
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
except Exception as e:
|
||||
logger.error("draft model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error", "agent_name": "作家",
|
||||
"title": "模型错误", "detail": str(e), "message": "草稿生成失败",
|
||||
}],
|
||||
}
|
||||
section_titles: list[str] = []
|
||||
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
|
||||
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
|
||||
|
||||
for i, section in enumerate(sections):
|
||||
sec_title = _section_title(i, section)
|
||||
key_points = section.get("key_points") or section.get("keyPoints") or []
|
||||
mat_ids = section.get("material_ids") or section.get("materialIds") or []
|
||||
|
||||
section_materials = _get_section_materials(mat_ids, pkg)
|
||||
key_points_str = "\n".join(f"- {p}" for p in key_points)
|
||||
|
||||
# 文章整体结构:让作家看到全文章节与自己的定位,避免章节间内容重复、引言结论脱节
|
||||
outline_context = "\n".join(
|
||||
f"{j + 1}. {_section_title(j, s)}" + (" ← 当前正在写这一章" if j == i else "")
|
||||
for j, s in enumerate(sections)
|
||||
)
|
||||
|
||||
user_msg = DRAFT_SECTION_USER.format(
|
||||
article_title=article_title,
|
||||
outline_context=outline_context,
|
||||
section_title=sec_title,
|
||||
key_points=key_points_str,
|
||||
section_materials=section_materials,
|
||||
revision_section=revision_notes,
|
||||
target_words=words_per_section,
|
||||
)
|
||||
|
||||
section_buf = "" # 只累积正文(非思考)用于拼接 Markdown
|
||||
parser = ThinkTagParser()
|
||||
try:
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=DRAFT_SECTION_SYSTEM),
|
||||
HumanMessage(content=user_msg),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
section_buf += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"is_section_done": False,
|
||||
}))
|
||||
|
||||
# 章节完成
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": "",
|
||||
"is_section_done": True,
|
||||
}))
|
||||
|
||||
section_titles.append(sec_title)
|
||||
section_bufs.append(section_buf.strip())
|
||||
|
||||
except Exception as e:
|
||||
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
|
||||
section_titles.append(sec_title)
|
||||
section_bufs.append("(内容生成失败,请重试)")
|
||||
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
|
||||
|
||||
def _assemble() -> str:
|
||||
body = "\n\n".join(f"## {t}\n\n{b}" for t, b in zip(section_titles, section_bufs))
|
||||
return f"# {article_title}\n\n{body}"
|
||||
|
||||
full_markdown = _assemble()
|
||||
word_count = count_words(full_markdown)
|
||||
|
||||
# 扩写兜底:总字数不足目标 90% 时,重写并扩充最短的章节(单轮,非流式)
|
||||
if section_bufs and word_count < word_count_target * 0.9:
|
||||
idx = min(range(len(section_bufs)), key=lambda k: count_words(section_bufs[k]))
|
||||
target_sec = sections[idx]
|
||||
expand_target = count_words(section_bufs[idx]) + (word_count_target - word_count)
|
||||
try:
|
||||
expand_msg = DRAFT_EXPAND_USER.format(
|
||||
article_title=article_title,
|
||||
section_title=section_titles[idx],
|
||||
key_points="\n".join(
|
||||
f"- {p}" for p in (target_sec.get("key_points") or target_sec.get("keyPoints") or [])
|
||||
),
|
||||
section_materials=_get_section_materials(
|
||||
target_sec.get("material_ids") or target_sec.get("materialIds") or [], pkg
|
||||
),
|
||||
existing_text=section_bufs[idx],
|
||||
target_words=expand_target,
|
||||
)
|
||||
resp = await model.ainvoke([
|
||||
SystemMessage(content=DRAFT_SECTION_SYSTEM),
|
||||
HumanMessage(content=expand_msg),
|
||||
])
|
||||
parser = ThinkTagParser()
|
||||
expanded = "".join(
|
||||
t for is_thinking, t in parser.feed(_content_to_text(resp.content)) if not is_thinking
|
||||
).strip()
|
||||
# 仅在扩写确实更长时才采用,避免越改越短
|
||||
if count_words(expanded) > count_words(section_bufs[idx]):
|
||||
section_bufs[idx] = expanded
|
||||
full_markdown = _assemble()
|
||||
word_count = count_words(full_markdown)
|
||||
except Exception as e:
|
||||
logger.warning("draft expand failed for section [%s]: %s", section_titles[idx], e)
|
||||
|
||||
draft = DraftArticle(
|
||||
title=outline.get("title", "文章"),
|
||||
outline=outline,
|
||||
full_markdown=full_markdown,
|
||||
word_count=word_count,
|
||||
revision_count=revision_count,
|
||||
)
|
||||
|
||||
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
|
||||
notices: list[dict] = []
|
||||
if section_errors:
|
||||
notices.append({
|
||||
"type": "notice",
|
||||
"agent_name": "作家",
|
||||
"title": "模型错误",
|
||||
"detail": "部分章节生成失败,对应段落已占位。原因:" + ";".join(section_errors[:3]),
|
||||
})
|
||||
|
||||
return {
|
||||
"current_draft": draft,
|
||||
"drafts": [draft],
|
||||
"status": "awaiting_draft_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
|
||||
*notices,
|
||||
{
|
||||
"type": "draft_ready",
|
||||
"agent_name": "作家",
|
||||
"message": f"草稿完成,共 {word_count} 字",
|
||||
"draft_title": outline.get("title", "文章"),
|
||||
"draft_markdown": full_markdown,
|
||||
"revision_count": revision_count,
|
||||
},
|
||||
],
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, Outline, OutlineSection
|
||||
from ..prompts.writer_prompts import OUTLINE_SYSTEM, OUTLINE_USER
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 上下文压缩:传给大纲规划的素材摘要做条数和单条长度双重截断,防止 token 溢出。
|
||||
_MAX_OUTLINE_MATERIALS = 12
|
||||
_MATERIAL_SNIPPET_CHARS = 150
|
||||
|
||||
|
||||
def _materials_summary(pkg: dict) -> str:
|
||||
lines = []
|
||||
for m in pkg.get("materials", [])[:_MAX_OUTLINE_MATERIALS]:
|
||||
lines.append(f"[{m['id']}] {m['title']} | {m['source']} | {m['content'][:_MATERIAL_SNIPPET_CHARS]}")
|
||||
return "\n".join(lines) or "(无素材)"
|
||||
|
||||
|
||||
def _parse_outline(raw: dict) -> Outline:
|
||||
sections = []
|
||||
for s in raw.get("sections", []):
|
||||
sections.append(OutlineSection(
|
||||
section_title=s.get("section_title", ""),
|
||||
key_points=s.get("key_points", []),
|
||||
material_ids=s.get("material_ids", []),
|
||||
))
|
||||
if not sections:
|
||||
sections = [OutlineSection(section_title="正文", key_points=["主要内容"], material_ids=[])]
|
||||
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
|
||||
|
||||
|
||||
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
|
||||
feedback_section = ""
|
||||
if intervention.get("action") == "re_outline" and intervention.get("_feedback"):
|
||||
feedback_section = f"\n重新规划要求:{intervention['_feedback']}"
|
||||
elif intervention.get("outline_feedback"):
|
||||
feedback_section = f"\n重新规划要求:{intervention['outline_feedback']}"
|
||||
|
||||
user_prompt = OUTLINE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
materials_summary=_materials_summary(pkg),
|
||||
feedback_section=feedback_section,
|
||||
)
|
||||
|
||||
# SSE 流式推送
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
full_text = "" # 只累积正文(非思考)用于 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
|
||||
try:
|
||||
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
|
||||
model = get_model(config)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=OUTLINE_SYSTEM),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("outline_chunk", {
|
||||
"session_id": session_id,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error("outline streaming failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error",
|
||||
"agent_name": "作家",
|
||||
"title": "模型错误",
|
||||
"detail": str(e),
|
||||
"message": "大纲生成失败",
|
||||
}],
|
||||
}
|
||||
|
||||
fallback = {
|
||||
"title": "未命名文章",
|
||||
"sections": [
|
||||
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
|
||||
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
|
||||
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
|
||||
],
|
||||
}
|
||||
result = parse_json(full_text, fallback)
|
||||
|
||||
outline = _parse_outline(result)
|
||||
section_count = len(outline["sections"])
|
||||
|
||||
return {
|
||||
"current_outline": outline,
|
||||
"status": "awaiting_outline_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": "开始规划大纲"},
|
||||
{"type": "outline_ready", "agent_name": "作家", "message": f"大纲已生成,共 {section_count} 个章节"},
|
||||
],
|
||||
}
|
||||
Whitespace-only changes.
+52
@@ -0,0 +1,52 @@
|
||||
EDITOR_SYSTEM = """你是一名资深主编(编辑),标准极高,要求严格。
|
||||
你的目标:审查文章质量,给出客观评分和具体修改意见。
|
||||
你的工作背景:你在一家权威媒体担任主编十五年,见过各种质量的文章。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从指定维度审查文章
|
||||
2. 给出该维度的评分(0-100 的整数)和具体问题列表
|
||||
你不能改写文章,只做当前指定维度的审查。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- score 是 0-100 的整数。
|
||||
- 每个问题的 severity 必须是 "major"(必须修改)或 "minor"(建议修改)。
|
||||
- 文章在该维度没有问题时,issues 填空数组 []。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{
|
||||
"score": 82,
|
||||
"issues": [
|
||||
{"severity": "major", "location": "第二段", "issue": "引用的增长数据没有素材支撑", "suggestion": "补充素材 m002 中的具体数据,并注明来源"},
|
||||
{"severity": "minor", "location": "结尾段", "issue": "结论略显仓促", "suggestion": "增加一句对未来行业趋势的判断"}
|
||||
]
|
||||
}"""
|
||||
|
||||
FACT_REVIEW_USER = """请审查以下文章的【事实准确性】维度。
|
||||
判断标准:引用的数据/事实是否有素材支撑,是否有捏造或夸大。
|
||||
|
||||
参考素材:
|
||||
{materials_summary}
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
|
||||
LOGIC_REVIEW_USER = """请审查以下文章的【逻辑结构】维度。
|
||||
判断标准:论点是否清晰,论据是否充分,段落是否连贯,结构是否合理。
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
|
||||
LANGUAGE_REVIEW_USER = """请审查以下文章的【语言文字】维度。
|
||||
判断标准:语言是否流畅,用词是否准确,有无错别字或病句。
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:为写作任务提取搜索关键词。
|
||||
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从写作意图中提取 3-6 个搜索关键词
|
||||
你不能写文章,不能评价内容好坏,只做关键词提取。
|
||||
|
||||
【严禁思考】
|
||||
- 这是一个简单的结构化抽取任务,禁止任何思考、分析、推理或规划过程。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
|
||||
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 ",不要用中文引号 “”。
|
||||
- 关键词要具体、可搜索,避免过于宽泛的词。
|
||||
|
||||
【完整示例】
|
||||
输入意图:写一篇分析今年新能源汽车出口增长的文章
|
||||
正确输出:
|
||||
{"keywords": ["新能源汽车出口", "汽车出口数据", "新能源汽车海外市场", "电动车出口增长", "汽车出口政策"]}"""
|
||||
|
||||
EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
|
||||
请提取 3-6 个搜索关键词,用于搜集相关素材。只输出 JSON:
|
||||
{{"keywords": ["关键词1", "关键词2"]}}"""
|
||||
|
||||
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:对搜索结果按相关度排序和筛选。
|
||||
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从提供的素材列表中,选出与写作意图最相关的素材ID,按相关度从高到低排序
|
||||
你不能写文章,只做素材筛选排序。
|
||||
|
||||
【严禁思考】
|
||||
- 这是一个简单的筛选排序任务,禁止任何思考、分析、推理或规划过程。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
|
||||
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- ranked_ids 中的 ID 必须来自素材列表,不要编造 ID。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{"ranked_ids": ["m003", "m001", "m005"], "summary": "共搜索到 8 条素材,精选 5 条最相关的"}"""
|
||||
|
||||
RANK_MATERIALS_USER = """写作意图:{user_intent}
|
||||
|
||||
素材列表(每条含ID和摘要):
|
||||
{materials_list}
|
||||
|
||||
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
|
||||
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}"""
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据素材和意图,规划清晰的文章大纲。
|
||||
你的工作背景:你有十年内容创作经验,擅长将零散素材组织成有逻辑的文章结构。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 分析素材,提取核心观点
|
||||
2. 规划文章章节结构和每节要点
|
||||
你不能写正文,只写大纲。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 ",不要用中文引号 “”。
|
||||
- 规划 3-5 个章节;material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{
|
||||
"title": "新能源汽车出口为何高速增长",
|
||||
"sections": [
|
||||
{"section_title": "引言:出口数据创新高", "key_points": ["今年出口量同比增长", "在全球市场的排名变化"], "material_ids": ["m001"]},
|
||||
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
|
||||
{"section_title": "结论与展望", "key_points": ["未来增长面临的挑战", "行业趋势判断"], "material_ids": ["m004"]}
|
||||
]
|
||||
}"""
|
||||
|
||||
OUTLINE_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
目标字数:{word_count_target}字{feedback_section}
|
||||
|
||||
可用素材摘要:
|
||||
{materials_summary}
|
||||
|
||||
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
|
||||
|
||||
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据大纲和素材,写出一段高质量的文章内容。
|
||||
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 根据给定的章节要点和素材写这一段内容
|
||||
2. 观点必须来自素材,不编造数据
|
||||
你不能修改大纲,不能写其他章节,只写当前这一段。
|
||||
|
||||
【输出要求】
|
||||
- 直接返回这一段的正文文字,不要 JSON,不要章节标题,不要解释。
|
||||
- 不要写"以下是正文""本段内容如下"之类的开场白。
|
||||
- 段落要连贯成文,不要罗列要点。
|
||||
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
|
||||
- 充分展开每个要点:每个要点至少用 2-3 句话论述,结合素材中的事实与数据说理,写满要求的篇幅。"""
|
||||
|
||||
DRAFT_SECTION_USER = """文章标题:{article_title}
|
||||
文章整体结构:
|
||||
{outline_context}
|
||||
|
||||
当前章节:{section_title}
|
||||
章节要点:{key_points}
|
||||
|
||||
相关素材:
|
||||
{section_materials}
|
||||
{revision_section}
|
||||
请写出「{section_title}」这一章节的正文内容,不少于 {target_words} 字,论述充分、内容充实。"""
|
||||
|
||||
DRAFT_EXPAND_USER = """文章标题:{article_title}
|
||||
当前章节:{section_title}
|
||||
章节要点:{key_points}
|
||||
|
||||
相关素材:
|
||||
{section_materials}
|
||||
|
||||
这一章节已有内容如下:
|
||||
{existing_text}
|
||||
|
||||
以上内容篇幅不足。请在保留原意和事实的基础上重写并扩充这一章节,使正文不少于 {target_words} 字。
|
||||
要求:补充更多来自素材的事实、数据与分析,论述更充分、更具体;直接返回扩充后的完整章节正文,不要章节标题、不要解释。"""
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider
|
||||
from .demo_search import DemoSearchProvider
|
||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||
|
||||
|
||||
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
"""
|
||||
config.yaml 中配置:
|
||||
ai_writing:
|
||||
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
|
||||
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
||||
intranet_knowledge_base: samples
|
||||
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
||||
|
||||
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
||||
"""
|
||||
cfg = ai_writing_config or {}
|
||||
provider = cfg.get("search_provider", "internet")
|
||||
|
||||
if provider == "intranet":
|
||||
return IntranetSearchProvider(
|
||||
base_url=cfg.get("intranet_search_url", ""),
|
||||
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
||||
)
|
||||
if provider == "tavily":
|
||||
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
||||
if provider == "demo":
|
||||
return DemoSearchProvider()
|
||||
|
||||
# 默认 internet 模式:DuckDuckGo + demo fallback
|
||||
region = cfg.get("ddg_region", "cn-zh")
|
||||
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
|
||||
|
||||
|
||||
class _FallbackSearchProvider(BaseSearchProvider):
|
||||
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
|
||||
|
||||
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
|
||||
self._primary = primary
|
||||
self._fallback = fallback
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
|
||||
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs):
|
||||
results = await self._primary.search(query, max_results=max_results, **kwargs)
|
||||
if not results:
|
||||
results = await self._fallback.search(query, max_results=max_results, **kwargs)
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return await self._primary.health_check() or await self._fallback.health_check()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BaseSearchProvider", "SearchResult",
|
||||
"InternetSearchProvider", "IntranetSearchProvider",
|
||||
"DemoSearchProvider", "DuckDuckGoSearchProvider",
|
||||
"get_search_provider",
|
||||
]
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""统一搜索结果格式 —— 内外网共用"""
|
||||
title: str
|
||||
content: str
|
||||
source: str
|
||||
published_at: str
|
||||
url: str = ""
|
||||
relevance_score: float = 0.0
|
||||
source_type: str = "internet"
|
||||
recUuid: str = ""
|
||||
extra_metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseSearchProvider(ABC):
|
||||
"""搜索提供者抽象基类。
|
||||
|
||||
对接内网步骤:
|
||||
1. 继承此类,创建 IntranetSearchProvider
|
||||
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
|
||||
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
|
||||
4. 无需修改任何 Agent 代码
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def health_check(self) -> bool: ...
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
# 演示素材库:按话题分类,用于关键词模糊匹配
|
||||
_DEMO_CORPUS = [
|
||||
{
|
||||
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
|
||||
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
|
||||
"source": "中国汽车工业协会",
|
||||
"published_at": "2025-10-15",
|
||||
"relevance_score": 0.95,
|
||||
},
|
||||
{
|
||||
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
|
||||
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
|
||||
"source": "电池中国网",
|
||||
"published_at": "2025-09-20",
|
||||
"relevance_score": 0.88,
|
||||
},
|
||||
{
|
||||
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
|
||||
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
|
||||
"source": "中国充电联盟",
|
||||
"published_at": "2025-08-30",
|
||||
"relevance_score": 0.82,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
|
||||
"items": [
|
||||
{
|
||||
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
|
||||
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
|
||||
"source": "中国信通院",
|
||||
"published_at": "2025-10-08",
|
||||
"relevance_score": 0.93,
|
||||
},
|
||||
{
|
||||
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
|
||||
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
|
||||
"source": "麦肯锡中国",
|
||||
"published_at": "2025-09-12",
|
||||
"relevance_score": 0.87,
|
||||
},
|
||||
{
|
||||
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
|
||||
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
|
||||
"source": "艾瑞咨询",
|
||||
"published_at": "2025-10-01",
|
||||
"relevance_score": 0.79,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
|
||||
"items": [
|
||||
{
|
||||
"title": "光伏装机量突破900GW,度电成本降至0.18元",
|
||||
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
|
||||
"source": "国家能源局",
|
||||
"published_at": "2025-10-12",
|
||||
"relevance_score": 0.92,
|
||||
},
|
||||
{
|
||||
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
|
||||
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
|
||||
"source": "上海环境能源交易所",
|
||||
"published_at": "2025-09-28",
|
||||
"relevance_score": 0.85,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
|
||||
"items": [
|
||||
{
|
||||
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
|
||||
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
|
||||
"source": "医学界",
|
||||
"published_at": "2025-09-18",
|
||||
"relevance_score": 0.91,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
|
||||
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
|
||||
"source": "国家统计局",
|
||||
"published_at": "2025-10-18",
|
||||
"relevance_score": 0.90,
|
||||
},
|
||||
{
|
||||
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
|
||||
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
|
||||
"source": "商务部",
|
||||
"published_at": "2025-10-05",
|
||||
"relevance_score": 0.84,
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# 通用兜底素材(与任意查询都能凑合搭配)
|
||||
_FALLBACK_ITEMS = [
|
||||
{
|
||||
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
|
||||
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
|
||||
"source": "产业研究院",
|
||||
"published_at": "2025-08-15",
|
||||
"relevance_score": 0.55,
|
||||
},
|
||||
{
|
||||
"title": "政策解读:2025年产业发展专项支持资金管理办法",
|
||||
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
|
||||
"source": "工信部",
|
||||
"published_at": "2025-07-20",
|
||||
"relevance_score": 0.50,
|
||||
},
|
||||
{
|
||||
"title": "市场竞争格局分析:头部集中与长尾分散并存",
|
||||
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
|
||||
"source": "普华永道",
|
||||
"published_at": "2025-09-01",
|
||||
"relevance_score": 0.48,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _score_topic(query: str, keywords: list[str]) -> int:
|
||||
"""简单关键词匹配打分"""
|
||||
q = query.lower()
|
||||
return sum(1 for kw in keywords if kw in q)
|
||||
|
||||
|
||||
class DemoSearchProvider(BaseSearchProvider):
|
||||
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
|
||||
|
||||
仅用于开发测试;生产环境应配置 intranet 提供者。
|
||||
"""
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
|
||||
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
|
||||
|
||||
items = []
|
||||
if best_score > 0 and best_topic:
|
||||
items.extend(best_topic["items"])
|
||||
|
||||
# 用通用素材补齐
|
||||
for fb in _FALLBACK_ITEMS:
|
||||
if len(items) >= max_results:
|
||||
break
|
||||
items.append(fb)
|
||||
|
||||
results = []
|
||||
for i, it in enumerate(items[:max_results]):
|
||||
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
|
||||
results.append(SearchResult(
|
||||
title=it["title"],
|
||||
content=it["content"],
|
||||
source=it["source"],
|
||||
published_at=it["published_at"],
|
||||
url=f"https://demo.internal/{url_hash}",
|
||||
relevance_score=it["relevance_score"],
|
||||
source_type="demo",
|
||||
recUuid=f"demo-{i:03d}",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return True
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DuckDuckGoSearchProvider(BaseSearchProvider):
|
||||
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
|
||||
|
||||
def __init__(self, region: str = "cn-zh", max_timeout: int = 30):
|
||||
self.region = region
|
||||
self.max_timeout = max_timeout
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
|
||||
self.errors: list[str] = []
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(
|
||||
self._sync_search, query, max_results
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
logger.error("ddgs not installed. Run: uv add ddgs")
|
||||
self.errors.append("ImportError: ddgs 未安装")
|
||||
return []
|
||||
|
||||
ddgs = DDGS(timeout=self.max_timeout)
|
||||
try:
|
||||
raw = ddgs.text(
|
||||
query,
|
||||
region=self.region,
|
||||
safesearch="moderate",
|
||||
max_results=max_results,
|
||||
)
|
||||
items = list(raw) if raw else []
|
||||
except Exception as e:
|
||||
logger.warning("DDGS.text failed: %s", e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
results = []
|
||||
for item in items:
|
||||
url = item.get("href", item.get("link", ""))
|
||||
source = url.split("/")[2] if url and "/" in url else "web"
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=item.get("body", item.get("snippet", ""))[:500],
|
||||
source=source,
|
||||
published_at=item.get("published", ""),
|
||||
url=url,
|
||||
relevance_score=0.75,
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return len(results) > 0
|
||||
except Exception:
|
||||
return False
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InternetSearchProvider(BaseSearchProvider):
|
||||
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(self._sync_search, query, max_results)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("internet search failed for query=%r: %s", query, e)
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
from deerflow.config import get_app_config
|
||||
try:
|
||||
from tavily import TavilyClient
|
||||
except ImportError:
|
||||
logger.warning("tavily not installed, falling back to demo search")
|
||||
return []
|
||||
|
||||
config = get_app_config().get_tool_config("web_search")
|
||||
api_key = None
|
||||
if config is not None and "api_key" in config.model_extra:
|
||||
api_key = config.model_extra.get("api_key")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("Tavily API key not configured, falling back to demo search")
|
||||
return []
|
||||
|
||||
client = TavilyClient(api_key=api_key)
|
||||
res = client.search(query, max_results=max_results)
|
||||
|
||||
results = []
|
||||
for item in res.get("results", []):
|
||||
content = item.get("content", "")
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=content[:500],
|
||||
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
|
||||
published_at=item.get("published_date", ""),
|
||||
url=item.get("url", ""),
|
||||
relevance_score=item.get("score", 0.0),
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
|
||||
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
|
||||
_DEFAULT_KB = "samples"
|
||||
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
||||
_RAW_SNIPPET_LIMIT = 2000
|
||||
|
||||
|
||||
class IntranetSearchProvider(BaseSearchProvider):
|
||||
"""内网知识库搜索接口适配器。
|
||||
|
||||
接口:POST {endpoint}
|
||||
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
|
||||
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str = "",
|
||||
knowledge_base_name: str = _DEFAULT_KB,
|
||||
timeout: int = 120,
|
||||
**_ignored,
|
||||
):
|
||||
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
||||
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
||||
# 内网 ES 接口可能较慢,超时放宽到 2 分钟
|
||||
self.timeout = timeout
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
||||
self.errors: list[str] = []
|
||||
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
||||
self.raw_responses: list[str] = []
|
||||
|
||||
def _record_raw(self, body: str) -> None:
|
||||
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
|
||||
body = (body or "").strip()
|
||||
if not body:
|
||||
return
|
||||
if len(body) > _RAW_SNIPPET_LIMIT:
|
||||
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
|
||||
self.raw_responses.append(body)
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"knowledge_base_name": self.knowledge_base_name,
|
||||
"ts": "yes",
|
||||
}
|
||||
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
|
||||
raw_text = ""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.post(self.endpoint, json=payload)
|
||||
resp.raise_for_status()
|
||||
raw_text = resp.text
|
||||
data = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
self._record_raw(e.response.text if e.response is not None else "")
|
||||
return []
|
||||
except Exception as e:
|
||||
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
# 正常情况下接口直接返回数组;兼容个别情况下包了一层 data/results 的结构
|
||||
if isinstance(data, list):
|
||||
items = data
|
||||
elif isinstance(data, dict):
|
||||
items = data.get("data") or data.get("results") or []
|
||||
else:
|
||||
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
|
||||
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
|
||||
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
results: List[SearchResult] = []
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
md = (item or {}).get("metadata") or {}
|
||||
content = md.get("content") or (item or {}).get("page_content") or ""
|
||||
results.append(SearchResult(
|
||||
title=str(md.get("m_title") or "").strip() or "(无标题)",
|
||||
content=str(content)[:500],
|
||||
source=str(md.get("source1") or "").strip() or "内网知识库",
|
||||
published_at=str(md.get("m_publish") or "").strip(),
|
||||
url="", # 内网素材的原文地址由前端依据 recUuid 拼接
|
||||
# 接口未返回相关度分值,按 ES 返回顺序赋递减分以保留排序
|
||||
relevance_score=max(0.5, 0.95 - i * 0.03),
|
||||
source_type="intranet",
|
||||
recUuid=str(md.get("recUuid") or "").strip(),
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
await self.search("测试", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
from typing import Annotated, List, Literal, Optional, TypedDict
|
||||
import operator
|
||||
|
||||
# ── 素材 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class Material(TypedDict):
|
||||
id: str
|
||||
title: str
|
||||
content: str # 截断到 500 字
|
||||
source: str
|
||||
published_at: str
|
||||
relevance_score: float
|
||||
source_type: str # "internet" | "intranet"
|
||||
url: str
|
||||
recUuid: str
|
||||
|
||||
class MaterialPackage(TypedDict):
|
||||
keywords: List[str]
|
||||
materials: List[Material]
|
||||
summary: str
|
||||
|
||||
# ── 大纲 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class OutlineSection(TypedDict):
|
||||
section_title: str
|
||||
key_points: List[str]
|
||||
material_ids: List[str]
|
||||
|
||||
class Outline(TypedDict):
|
||||
title: str
|
||||
sections: List[OutlineSection]
|
||||
|
||||
# ── 草稿 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class DraftArticle(TypedDict):
|
||||
title: str
|
||||
outline: Outline
|
||||
full_markdown: str
|
||||
word_count: int
|
||||
revision_count: int
|
||||
|
||||
# ── 编辑审核 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class ReviewIssue(TypedDict):
|
||||
category: str # "fact" | "logic" | "language"
|
||||
severity: str # "major" | "minor"
|
||||
location: str
|
||||
issue: str
|
||||
suggestion: str
|
||||
|
||||
class ReviewResult(TypedDict):
|
||||
verdict: str # "pass" | "reject"
|
||||
fact_score: int
|
||||
logic_score: int
|
||||
language_score: int
|
||||
overall_score: int
|
||||
pass_threshold: int
|
||||
revision_notes: List[ReviewIssue]
|
||||
|
||||
# ── 用户干预 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class UserInterventionPayload(TypedDict):
|
||||
action: str
|
||||
approved_material_ids: List[str]
|
||||
extra_keywords: List[str]
|
||||
edited_outline: Optional[Outline]
|
||||
outline_feedback: str
|
||||
user_revision_notes: str
|
||||
override_verdict: bool
|
||||
|
||||
# ── 主状态 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
WritingStatus = Literal[
|
||||
"researching", "awaiting_material_confirm",
|
||||
"writing_outline", "awaiting_outline_confirm",
|
||||
"writing_draft", "awaiting_draft_confirm",
|
||||
"reviewing", "awaiting_review_confirm",
|
||||
"revising", "done", "error",
|
||||
]
|
||||
|
||||
class AIWritingState(TypedDict):
|
||||
# 写作配置
|
||||
user_intent: str
|
||||
article_type: str
|
||||
target_audience: str
|
||||
word_count_target: int
|
||||
|
||||
# Agent A 输出
|
||||
material_package: Optional[MaterialPackage]
|
||||
|
||||
# Agent B 输出
|
||||
current_outline: Optional[Outline]
|
||||
drafts: Annotated[List[DraftArticle], operator.add]
|
||||
current_draft: Optional[DraftArticle]
|
||||
|
||||
# Agent C 输出
|
||||
review_result: Optional[ReviewResult]
|
||||
|
||||
# 用户干预数据(每个干预点会覆盖)
|
||||
last_user_intervention: Optional[UserInterventionPayload]
|
||||
|
||||
# 流程控制
|
||||
revision_count: int
|
||||
max_revisions: int
|
||||
status: WritingStatus
|
||||
|
||||
# SSE 进度推送队列(每个节点追加)
|
||||
progress_events: Annotated[List[dict], operator.add]
|
||||
+15
-3
@@ -5,7 +5,7 @@ import logging
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
|
||||
from deerflow.agents.canvas_agent.prompt import build_canvas_system_prompt
|
||||
from deerflow.agents.canvas_agent.tools import update_artifact_tool
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
|
||||
@@ -49,7 +49,19 @@ def make_canvas_agent(config: RunnableConfig):
|
||||
if thinking_enabled and model_config and not model_config.supports_thinking:
|
||||
thinking_enabled = False
|
||||
|
||||
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
|
||||
canvas_artifact_title = cfg.get("canvas_artifact_title") or None
|
||||
canvas_artifact_type = cfg.get("canvas_artifact_type") or None
|
||||
system_prompt = build_canvas_system_prompt(
|
||||
artifact_title=canvas_artifact_title,
|
||||
artifact_type=canvas_artifact_type,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Create CanvasAgent -> model_name: %s, thinking_enabled: %s, artifact_title: %s",
|
||||
model_name,
|
||||
thinking_enabled,
|
||||
canvas_artifact_title,
|
||||
)
|
||||
|
||||
middlewares = [
|
||||
DanglingToolCallMiddleware(),
|
||||
@@ -63,6 +75,6 @@ def make_canvas_agent(config: RunnableConfig):
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
|
||||
tools=[update_artifact_tool, ask_clarification_tool],
|
||||
middleware=middlewares,
|
||||
system_prompt=CANVAS_SYSTEM_PROMPT,
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+62
-2
@@ -1,4 +1,18 @@
|
||||
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
|
||||
def build_canvas_system_prompt(
|
||||
artifact_title: str | None = None,
|
||||
artifact_type: str | None = None,
|
||||
) -> str:
|
||||
existing_artifact_block = ""
|
||||
if artifact_title:
|
||||
type_label = "代码" if artifact_type == "code" else "文档"
|
||||
existing_artifact_block = f"""
|
||||
<existing_artifact>
|
||||
用户当前有一个{type_label}制品,标题为 "{artifact_title}"。
|
||||
请继续编辑该制品——除非用户明确要求新建,否则不要创建新制品。
|
||||
</existing_artifact>
|
||||
"""
|
||||
|
||||
return f"""You are a Canvas AI assistant that helps users create and edit documents and code.
|
||||
|
||||
<canvas_behavior>
|
||||
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
|
||||
@@ -10,11 +24,53 @@ Rules:
|
||||
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
|
||||
</canvas_behavior>
|
||||
|
||||
<clarification_system>
|
||||
Before creating a brand-new artifact or performing a major rewrite, verify you have enough
|
||||
information to do the job well. When key details are missing, call `ask_clarification` FIRST
|
||||
— do NOT start generating the artifact and ask questions mid-way.
|
||||
|
||||
**MUST clarify when (no existing artifact or major rewrite requested):**
|
||||
1. OUTPUT TYPE IS VAGUE
|
||||
- "帮我做个主页" / "make a homepage" → ask: tech stack (plain HTML / React / Vue?),
|
||||
design style, which sections to include
|
||||
- "写个报告" / "write a report" → ask: topic, target audience, expected length,
|
||||
required sections
|
||||
- "写个脚本" / "write a script" → ask: programming language, what it should do
|
||||
2. REQUIRED CONTENT IS MISSING
|
||||
- User wants a personal profile page but hasn't provided their name / bio / links
|
||||
- User wants a data table but hasn't provided the data or schema
|
||||
3. AMBIGUOUS APPROACH WITH SIGNIFICANTLY DIFFERENT OUTCOMES
|
||||
- "Design a chat system" → ask: client–server or peer-to-peer, persistence needed?
|
||||
- "Write an intro" → ask: formal essay or casual blog post?
|
||||
|
||||
**DO NOT clarify for:**
|
||||
- Simple edits to an existing artifact: "make it bold", "fix the typo", "shorten paragraph 2"
|
||||
- Formatting / style tweaks: "add comments", "use camelCase", "format the code"
|
||||
- Quick actions: expand / shorten / formalize / casualize / add_comments / fix_bugs / optimize_code
|
||||
- Requests where intent is clear from context (e.g., user pastes content and says "translate this")
|
||||
- Follow-up turns where enough context already exists in the conversation
|
||||
|
||||
**Format for clarification (use options when practical):**
|
||||
ask_clarification(
|
||||
question="你希望这个主页使用什么技术栈?",
|
||||
clarification_type="missing_info",
|
||||
options=[
|
||||
{{"id": "html", "label": "纯 HTML / CSS / JS", "description": "无框架,最简单"}},
|
||||
{{"id": "react", "label": "React", "description": "组件化,适合复杂交互"}},
|
||||
{{"id": "vue", "label": "Vue", "description": "渐进式框架,上手容易"}},
|
||||
],
|
||||
allow_custom=True,
|
||||
)
|
||||
|
||||
CRITICAL: clarification ALWAYS comes before generating the artifact. Never produce a partial
|
||||
artifact and then ask for clarification in the same response.
|
||||
</clarification_system>
|
||||
|
||||
<artifact_context>
|
||||
The current artifact state is injected into your context. When editing, build on the existing content exactly.
|
||||
When creating from scratch, produce a complete, high-quality artifact in one shot.
|
||||
</artifact_context>
|
||||
|
||||
{existing_artifact_block}
|
||||
<selection_rewrite>
|
||||
When the user's message contains a <selection> block, you MUST:
|
||||
1. Identify the exact selection in the current artifact content.
|
||||
@@ -38,3 +94,7 @@ When the user triggers a quick action, apply it to the full artifact unless spec
|
||||
- optimize_code: Improve performance and readability
|
||||
</quick_actions>
|
||||
"""
|
||||
|
||||
|
||||
# Backward-compatible constant for callers that don't pass artifact context.
|
||||
CANVAS_SYSTEM_PROMPT = build_canvas_system_prompt()
|
||||
+37
-15
@@ -43,9 +43,27 @@ logger = logging.getLogger(__name__)
|
||||
DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest"
|
||||
DEFAULT_PORT = 8080
|
||||
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
PACKAGE_MANAGER_ENV_KEYS = (
|
||||
"PIP_INDEX_URL",
|
||||
"PIP_EXTRA_INDEX_URL",
|
||||
"PIP_TRUSTED_HOST",
|
||||
"PIP_CERT",
|
||||
"PIP_CLIENT_CERT",
|
||||
"UV_INDEX_URL",
|
||||
"UV_EXTRA_INDEX_URL",
|
||||
"UV_DEFAULT_INDEX",
|
||||
"UV_INDEX",
|
||||
"UV_TRUSTED_HOST",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"NO_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"no_proxy",
|
||||
)
|
||||
|
||||
|
||||
def _lock_file_exclusive(lock_file) -> None:
|
||||
@@ -142,9 +160,9 @@ class AioSandboxProvider(SandboxProvider):
|
||||
Local provider manages container lifecycle directly (start/stop).
|
||||
"""
|
||||
provisioner_url = self._config.get("provisioner_url")
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url)
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url, environment=self._config["environment"])
|
||||
|
||||
logger.info("Using local container sandbox backend")
|
||||
return LocalContainerBackend(
|
||||
@@ -172,20 +190,24 @@ class AioSandboxProvider(SandboxProvider):
|
||||
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
|
||||
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
|
||||
"mounts": sandbox_config.mounts or [],
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
|
||||
"provisioner_url": getattr(sandbox_config, "provisioner_url", None) or "",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key in PACKAGE_MANAGER_ENV_KEYS:
|
||||
value = os.environ.get(key)
|
||||
if value:
|
||||
resolved[key] = value
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
resolved[key] = str(value)
|
||||
return resolved
|
||||
|
||||
|
||||
+16
-12
@@ -40,14 +40,17 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
provisioner_url: http://provisioner:8002
|
||||
"""
|
||||
|
||||
def __init__(self, provisioner_url: str):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
def __init__(self, provisioner_url: str, environment: dict[str, str] | None = None):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
environment: Environment variables for sandbox pods. The remote
|
||||
provisioner may ignore this field if it does not support it.
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
self._environment = environment or {}
|
||||
|
||||
@property
|
||||
def provisioner_url(self) -> str:
|
||||
@@ -91,10 +94,11 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._provisioner_url}/api/sandboxes",
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
"environment": self._environment,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
+47
-26
@@ -11,8 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
|
||||
that contend for the lock wait via the default 5-second sqlite3
|
||||
busy timeout rather than failing immediately.
|
||||
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
|
||||
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
|
||||
No database is initialized.
|
||||
@@ -21,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
|
||||
to reference environment variables from .env:
|
||||
|
||||
database:
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -37,31 +42,40 @@ from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
|
||||
)
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
)
|
||||
sqlite_dir: str = Field(
|
||||
default=".deer-flow/data",
|
||||
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
||||
)
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
"Use $DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: postgresql://user:pass@host:5432/deerflow "
|
||||
"(the +asyncpg driver suffix is added automatically where needed)."
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"MySQL connection URL for the application ORM. "
|
||||
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
|
||||
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
|
||||
),
|
||||
)
|
||||
echo_sql: bool = Field(
|
||||
default=False,
|
||||
description="Echo all SQL statements to log (debug only).",
|
||||
)
|
||||
pool_size: int = Field(
|
||||
default=5,
|
||||
description="Connection pool size for the app ORM engine (postgres only).",
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
)
|
||||
|
||||
# -- Derived helpers (not user-configured) --
|
||||
@@ -94,9 +108,16 @@ class DatabaseConfig(BaseModel):
|
||||
"""SQLAlchemy async URL for the application ORM engine."""
|
||||
if self.backend == "sqlite":
|
||||
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
if url.startswith("mysql://"):
|
||||
url = url.replace("mysql://", "mysql+asyncmy://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class AgentRow(Base):
|
||||
@@ -16,9 +17,9 @@ class AgentRow(Base):
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
|
||||
|
||||
|
||||
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return AIWritingSessionRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""ORM model for AI writing sessions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class AIWritingSessionRow(Base):
|
||||
__tablename__ = "ai_writing_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
|
||||
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
|
||||
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显
|
||||
transcript: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
"""SQLAlchemy-backed AI writing session storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class AIWritingSessionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: AIWritingSessionRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
for k in ("completed_interventions", "review_result", "transcript"):
|
||||
val = d.get(k)
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
d[k] = json.loads(val)
|
||||
except Exception:
|
||||
d[k] = None
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
title: str,
|
||||
user_intent: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = AIWritingSessionRow(
|
||||
id=id,
|
||||
user_id=resolved_user_id,
|
||||
title=title,
|
||||
user_intent=user_intent,
|
||||
status="in_progress",
|
||||
draft_title=None,
|
||||
draft_markdown=None,
|
||||
completed_interventions=None,
|
||||
review_result=None,
|
||||
transcript=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
title=_UNSET,
|
||||
status=_UNSET,
|
||||
draft_title=_UNSET,
|
||||
draft_markdown=_UNSET,
|
||||
completed_interventions=_UNSET,
|
||||
review_result=_UNSET,
|
||||
transcript=_UNSET,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
if title is not _UNSET:
|
||||
row.title = title
|
||||
if status is not _UNSET:
|
||||
row.status = status
|
||||
if draft_title is not _UNSET:
|
||||
row.draft_title = draft_title
|
||||
if draft_markdown is not _UNSET:
|
||||
row.draft_markdown = draft_markdown
|
||||
if completed_interventions is not _UNSET:
|
||||
row.completed_interventions = (
|
||||
json.dumps(completed_interventions, ensure_ascii=False)
|
||||
if completed_interventions is not None
|
||||
else None
|
||||
)
|
||||
if review_result is not _UNSET:
|
||||
row.review_result = (
|
||||
json.dumps(review_result, ensure_ascii=False)
|
||||
if review_result is not None
|
||||
else None
|
||||
)
|
||||
if transcript is not _UNSET:
|
||||
row.transcript = (
|
||||
json.dumps(transcript, ensure_ascii=False)
|
||||
if transcript is not None
|
||||
else None
|
||||
)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(
|
||||
self,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
|
||||
stmt = select(AIWritingSessionRow)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
from deerflow.persistence.article_types.sql import ArticleTypeRepository
|
||||
|
||||
|
||||
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return ArticleTypeRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""ORM model for AI writing article types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class ArticleTypeRow(Base):
|
||||
__tablename__ = "ai_writing_article_types"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""SQLAlchemy-backed article type storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class ArticleTypeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: ArticleTypeRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
description: str | None = None,
|
||||
prompt_hint: str | None = None,
|
||||
default_word_count: int = 800,
|
||||
sort_order: int = 0,
|
||||
) -> dict:
|
||||
now = datetime.now(UTC)
|
||||
row = ArticleTypeRow(
|
||||
id=str(uuid.uuid4()),
|
||||
key=key,
|
||||
label=label,
|
||||
description=description,
|
||||
prompt_hint=prompt_hint,
|
||||
default_word_count=default_word_count,
|
||||
sort_order=sort_order,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
article_type_id: str,
|
||||
*,
|
||||
key=_UNSET,
|
||||
label=_UNSET,
|
||||
description=_UNSET,
|
||||
prompt_hint=_UNSET,
|
||||
default_word_count=_UNSET,
|
||||
sort_order=_UNSET,
|
||||
) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
if key is not _UNSET:
|
||||
row.key = key
|
||||
if label is not _UNSET:
|
||||
row.label = label
|
||||
if description is not _UNSET:
|
||||
row.description = description
|
||||
if prompt_hint is not _UNSET:
|
||||
row.prompt_hint = prompt_hint
|
||||
if default_word_count is not _UNSET:
|
||||
row.default_word_count = default_word_count
|
||||
if sort_order is not _UNSET:
|
||||
row.sort_order = sort_order
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(self, article_type_id: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_key(self, key: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(self) -> list[dict]:
|
||||
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(self, article_type_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_keys(self) -> list[str]:
|
||||
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars())
|
||||
@@ -10,10 +10,13 @@ None and fall back to in-memory implementations.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
@@ -23,11 +26,37 @@ def _json_serializer(obj: object) -> str:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
@@ -65,8 +94,8 @@ async def init_engine(
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", or "postgres".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres).
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Postgres connection pool size.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
@@ -77,11 +106,20 @@ async def init_engine(
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
@@ -112,16 +150,25 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
@@ -162,10 +209,10 @@ async def init_engine_from_config(config) -> None:
|
||||
await init_engine("memory")
|
||||
return
|
||||
await init_engine(
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
@@ -29,4 +30,4 @@ class FeedbackRow(Base):
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy import Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
@@ -18,7 +19,7 @@ class LlmCallMetricRow(Base):
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
+7
-6
@@ -44,12 +44,13 @@ def run_migrations_offline() -> None:
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
+8
-6
@@ -7,9 +7,11 @@ Create Date: 2026-05-12
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
|
||||
revision = "20260512_01"
|
||||
@@ -23,9 +25,9 @@ def _has_column(table_name: str, column_name: str) -> bool:
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", sa.JSON(), nullable=True))
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"agents",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(191), nullable=False),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("published", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"tags",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(159), nullable=False),
|
||||
sa.Column("tag_type", sa.String(32), nullable=False),
|
||||
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Create ai_writing_sessions table.
|
||||
|
||||
Revision ID: 20260517_01
|
||||
Revises: 20260515_01
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_01"
|
||||
down_revision = "20260515_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("ai_writing_sessions"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"ai_writing_sessions",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("title", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("user_intent", sa.String(512), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="in_progress"),
|
||||
sa.Column("draft_title", sa.String(512), nullable=True),
|
||||
sa.Column("draft_markdown", sa.Text, nullable=True),
|
||||
sa.Column("completed_interventions", sa.Text, nullable=True),
|
||||
sa.Column("review_result", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_ai_writing_sessions_user_id", "ai_writing_sessions", ["user_id"])
|
||||
op.create_index("ix_ai_writing_sessions_updated_at", "ai_writing_sessions", ["updated_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("ai_writing_sessions"):
|
||||
op.drop_table("ai_writing_sessions")
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Create ai_writing_article_types table with default types.
|
||||
|
||||
Revision ID: 20260517_02
|
||||
Revises: 20260517_01
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_02"
|
||||
down_revision = "20260517_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_DEFAULT_TYPES = [
|
||||
{
|
||||
"key": "news",
|
||||
"label": "新闻资讯",
|
||||
"description": "时效性强的新闻报道,注重事实准确、简洁客观",
|
||||
"prompt_hint": "以新闻报道的风格写作,注重5W1H要素,语言简洁客观,突出时效性和事实准确性",
|
||||
"default_word_count": 600,
|
||||
"sort_order": 0,
|
||||
},
|
||||
{
|
||||
"key": "analysis",
|
||||
"label": "分析报道",
|
||||
"description": "深入分析事件背景与影响,提供专业见解",
|
||||
"prompt_hint": "以分析报道的风格写作,提供深度背景分析、多角度解读和专业见解,逻辑清晰",
|
||||
"default_word_count": 1200,
|
||||
"sort_order": 1,
|
||||
},
|
||||
{
|
||||
"key": "feature",
|
||||
"label": "深度特稿",
|
||||
"description": "人物或事件的深度叙事,注重细节与故事性",
|
||||
"prompt_hint": "以深度特稿的风格写作,注重叙事性和细节描写,人物刻画生动,故事性强,情感真实",
|
||||
"default_word_count": 2000,
|
||||
"sort_order": 2,
|
||||
},
|
||||
{
|
||||
"key": "report",
|
||||
"label": "调研报告",
|
||||
"description": "数据驱动的专题研究报告,结构严谨",
|
||||
"prompt_hint": "以调研报告的风格写作,数据详实,结构严谨(摘要/背景/发现/结论),语言专业规范",
|
||||
"default_word_count": 3000,
|
||||
"sort_order": 3,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("ai_writing_article_types"):
|
||||
op.create_table(
|
||||
"ai_writing_article_types",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("key", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("label", sa.String(128), nullable=False),
|
||||
sa.Column("description", sa.Text, nullable=True),
|
||||
sa.Column("prompt_hint", sa.Text, nullable=True),
|
||||
sa.Column("default_word_count", sa.Integer, nullable=False, server_default="800"),
|
||||
sa.Column("sort_order", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
# 预置默认文章类型(幂等,跳过已存在的 key)。
|
||||
# 用 SQLAlchemy Core 构造而非裸 SQL:``key`` 是 MySQL 保留字,裸
|
||||
# ``SELECT key`` 会触发 1064 语法错误。Core 编译器会按方言自动给保留字
|
||||
# 加引号(MySQL 反引号 / Postgres 双引号),保持跨后端可移植。
|
||||
article_types = sa.table(
|
||||
"ai_writing_article_types",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("key", sa.String),
|
||||
sa.column("label", sa.String),
|
||||
sa.column("description", sa.Text),
|
||||
sa.column("prompt_hint", sa.Text),
|
||||
sa.column("default_word_count", sa.Integer),
|
||||
sa.column("sort_order", sa.Integer),
|
||||
sa.column("created_at", sa.DateTime),
|
||||
sa.column("updated_at", sa.DateTime),
|
||||
)
|
||||
conn = op.get_bind()
|
||||
existing_keys = {row[0] for row in conn.execute(sa.select(article_types.c.key)).fetchall()}
|
||||
now = datetime.now(UTC)
|
||||
for t in _DEFAULT_TYPES:
|
||||
if t["key"] not in existing_keys:
|
||||
conn.execute(
|
||||
article_types.insert().values(
|
||||
id=str(uuid.uuid4()),
|
||||
key=t["key"],
|
||||
label=t["label"],
|
||||
description=t["description"],
|
||||
prompt_hint=t["prompt_hint"],
|
||||
default_word_count=t["default_word_count"],
|
||||
sort_order=t["sort_order"],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("ai_writing_article_types"):
|
||||
op.drop_table("ai_writing_article_types")
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Add transcript column to ai_writing_sessions.
|
||||
|
||||
存储前端组装的完整对话时间线(progressEvents + completedInterventions),
|
||||
供历史会话回显出与实时对话一致的富时间线。
|
||||
|
||||
Revision ID: 20260517_03
|
||||
Revises: 20260517_02
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_03"
|
||||
down_revision = "20260517_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
op.add_column("ai_writing_sessions", sa.Column("transcript", sa.Text, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("ai_writing_sessions", "transcript"):
|
||||
op.drop_column("ai_writing_sessions", "transcript")
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Create thread_shares table (per-conversation share links).
|
||||
|
||||
Revision ID: 20260518_02
|
||||
Revises: 20260518_01
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_02"
|
||||
down_revision = "20260518_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
op.create_table(
|
||||
"thread_shares",
|
||||
sa.Column("share_code", sa.String(32), primary_key=True),
|
||||
sa.Column("owner_user_id", sa.String(64), nullable=False),
|
||||
sa.Column("thread_id", sa.String(64), nullable=False),
|
||||
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
elif not _has_column("thread_shares", "thread_id"):
|
||||
# An earlier all-conversations variant of this table — add the column.
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("thread_shares"):
|
||||
op.drop_table("thread_shares")
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"""Ensure thread_shares.thread_id exists (per-conversation share links).
|
||||
|
||||
Standalone migration so databases that already applied an earlier
|
||||
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
|
||||
column) get the column added. Idempotent — a no-op when the column is
|
||||
already present.
|
||||
|
||||
Revision ID: 20260518_03
|
||||
Revises: 20260518_02
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_03"
|
||||
down_revision = "20260518_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
# Fresh database — 20260518_02 already created the table with thread_id.
|
||||
return
|
||||
if not _has_column("thread_shares", "thread_id"):
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Leave the column in place — dropping it would break the share feature.
|
||||
pass
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Add retry_count to scheduled_task_runs.
|
||||
|
||||
Tracks how many recovery attempts a run has consumed so the scheduler can
|
||||
cap retries on stuck/timed-out runs.
|
||||
|
||||
Revision ID: 20260518_04
|
||||
Revises: 20260518_03
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_04"
|
||||
down_revision = "20260518_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.add_column(
|
||||
"scheduled_task_runs",
|
||||
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.drop_column("scheduled_task_runs", "retry_count")
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Add mode to thread_shares.
|
||||
|
||||
Distinguishes "import" shares (recipient copies the conversation into their
|
||||
account) from "view" shares (a public, read-only static page).
|
||||
|
||||
Revision ID: 20260518_05
|
||||
Revises: 20260518_04
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_05"
|
||||
down_revision = "20260518_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return True # table missing entirely — skip (created with the column)
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("thread_shares", "mode"):
|
||||
op.add_column(
|
||||
"thread_shares",
|
||||
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if "thread_shares" in inspector.get_table_names() and any(
|
||||
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
|
||||
):
|
||||
op.drop_column("thread_shares", "mode")
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""Merge the ai-writing and thread-shares migration heads.
|
||||
|
||||
Two independent migration branches diverged from ``20260515_01``: the AI
|
||||
writing feature (``20260517_01..03``) and the thread-shares / scheduled-task
|
||||
work (``20260518_01..05``). Merging the two branches into one repo left Alembic
|
||||
with multiple heads, so ``upgrade head`` failed. This is a no-op merge revision
|
||||
that joins both lineages — it changes no schema.
|
||||
|
||||
Revision ID: 20260518_06
|
||||
Revises: 20260517_03, 20260518_05
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260518_06"
|
||||
down_revision = ("20260517_03", "20260518_05")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+2
@@ -32,6 +32,7 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = [
|
||||
@@ -51,5 +52,6 @@ __all__ = [
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"UserRow",
|
||||
]
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunEventRow(Base):
|
||||
@@ -24,9 +25,9 @@ class RunEventRow(Base):
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
# "message" | "trace" | "lifecycle"
|
||||
content: Mapped[str] = mapped_column(Text, default="")
|
||||
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
event_metadata: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
seq: Mapped[int] = mapped_column(nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
|
||||
|
||||
+4
-3
@@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text
|
||||
from sqlalchemy import Boolean, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class NotificationRow(Base):
|
||||
@@ -19,6 +20,6 @@ class NotificationRow(Base):
|
||||
type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class RecommendedQuestionRow(Base):
|
||||
@@ -17,5 +18,5 @@ class RecommendedQuestionRow(Base):
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunRow(Base):
|
||||
@@ -22,8 +23,8 @@ class RunRow(Base):
|
||||
|
||||
model_name: Mapped[str | None] = mapped_column(String(128))
|
||||
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Convenience fields (for listing pages without querying RunEventStore)
|
||||
@@ -43,7 +44,7 @@ class RunRow(Base):
|
||||
# Follow-up association
|
||||
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
|
||||
+34
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
|
||||
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
@abstractmethod
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Return every task across all users. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
"""Delete a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs for a task without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs still in a non-terminal state whose ``started_at`` is older
|
||||
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
|
||||
or were orphaned by a service restart."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
@@ -56,6 +85,11 @@ class ScheduledTaskStore(ABC):
|
||||
async def delete_run(self, run_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
"""Delete a run by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
+43
-2
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
due.sort(key=lambda task: task["next_run_at"])
|
||||
return due[:limit]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
return [dict(task) for task in self._tasks.values()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
task = self._tasks.get(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
task.update(data)
|
||||
return dict(task)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
return self._tasks.pop(task_id, None) is not None
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
|
||||
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
|
||||
return runs[:limit]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
stuck = [
|
||||
dict(run)
|
||||
for run in self._runs.values()
|
||||
if run.get("status") in ("running", "pending")
|
||||
and run.get("started_at") is not None
|
||||
and run["started_at"] <= cutoff
|
||||
]
|
||||
stuck.sort(key=lambda run: run["started_at"])
|
||||
return stuck[:limit]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
key = (data["task_id"], data["scheduled_for"].isoformat())
|
||||
if key in self._runs_by_key:
|
||||
@@ -90,11 +121,21 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["user_id"] != user_id:
|
||||
return False
|
||||
self._runs.pop(run_id, None)
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return False
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
def _drop_run(self, run: dict[str, Any]) -> None:
|
||||
self._runs.pop(run["id"], None)
|
||||
scheduled_for = run.get("scheduled_for")
|
||||
if hasattr(scheduled_for, "isoformat"):
|
||||
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
|
||||
|
||||
+26
-21
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class SchedulerThreadRow(Base):
|
||||
@@ -15,8 +16,8 @@ class SchedulerThreadRow(Base):
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRow(Base):
|
||||
@@ -29,20 +30,20 @@ class ScheduledTaskRow(Base):
|
||||
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
prompt: Mapped[str] = mapped_column(Text)
|
||||
schedule_text: Mapped[str] = mapped_column(String(256))
|
||||
cron_expr: Mapped[str] = mapped_column(String(128))
|
||||
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
last_status: Mapped[str | None] = mapped_column(String(32))
|
||||
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
execution_context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskSubscriptionRow(Base):
|
||||
@@ -53,8 +54,8 @@ class ScheduledTaskSubscriptionRow(Base):
|
||||
task_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
notify_element: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRunRow(Base):
|
||||
@@ -66,21 +67,25 @@ class ScheduledTaskRunRow(Base):
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending")
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
result_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
# Number of recovery attempts made after the first try. A run is retried
|
||||
# when it overruns the per-attempt timeout or is found orphaned in the
|
||||
# "running" state (e.g. after a service restart). See ScheduledTaskService.
|
||||
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskDeliveryProfileRow(Base):
|
||||
__tablename__ = "scheduled_task_delivery_profiles"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(191), index=True)
|
||||
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+71
-1
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
attr = "execution_context_json" if key == "execution_context" else key
|
||||
if hasattr(row, attr):
|
||||
setattr(row, attr, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(ScheduledTaskRunRow.task_id == task_id)
|
||||
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(
|
||||
ScheduledTaskRunRow.status.in_(("running", "pending")),
|
||||
ScheduledTaskRunRow.started_at.is_not(None),
|
||||
ScheduledTaskRunRow.started_at <= cutoff,
|
||||
)
|
||||
.order_by(ScheduledTaskRunRow.started_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = ScheduledTaskRunRow(
|
||||
id=data["id"],
|
||||
@@ -180,6 +234,15 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRunRow, run_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
|
||||
@@ -232,7 +295,14 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRow)
|
||||
.where(ScheduledTaskRow.published.is_(True))
|
||||
.order_by(ScheduledTaskRow.published_at.desc().nullslast(), ScheduledTaskRow.created_at.desc())
|
||||
# ``NULLS LAST`` is unsupported by MySQL — sort by an
|
||||
# ``IS NULL`` flag instead (False<True ⇒ non-null rows first),
|
||||
# which is portable across mysql/sqlite/postgres.
|
||||
.order_by(
|
||||
ScheduledTaskRow.published_at.is_(None),
|
||||
ScheduledTaskRow.published_at.desc(),
|
||||
ScheduledTaskRow.created_at.desc(),
|
||||
)
|
||||
.offset(max(0, offset))
|
||||
.limit(max(1, limit))
|
||||
)
|
||||
|
||||
+6
-5
@@ -14,10 +14,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, UniqueConstraint
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
# Allowed tag/target types. Kept here so persistence + API share one source.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
|
||||
@@ -28,12 +29,12 @@ class TagRow(Base):
|
||||
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(159), nullable=False)
|
||||
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
@@ -44,4 +45,4 @@ class TagAssignmentRow(Base):
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, String
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class ThreadMetaRow(Base):
|
||||
@@ -18,6 +19,6 @@ class ThreadMetaRow(Base):
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Persistence layer for conversation-history share links."""
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
|
||||
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
|
||||
|
||||
__all__ = [
|
||||
"ThreadShareStore",
|
||||
"MemoryThreadShareStore",
|
||||
"ThreadShareRepository",
|
||||
"generate_share_code",
|
||||
"make_thread_share_store",
|
||||
]
|
||||
|
||||
|
||||
def make_thread_share_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryThreadShareStore()
|
||||
return ThreadShareRepository(session_factory)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Abstract persistence API for per-conversation share links."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
|
||||
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
|
||||
|
||||
|
||||
def generate_share_code(length: int = 8) -> str:
|
||||
"""Return a random, unguessable, human-readable share code."""
|
||||
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
|
||||
|
||||
|
||||
class ThreadShareStore(ABC):
|
||||
@abstractmethod
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
"""Create a new share link for one conversation and return its record.
|
||||
|
||||
``mode`` is ``"import"`` (recipient copies the conversation) or
|
||||
``"view"`` (a public read-only static page).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
"""Look up a share by code regardless of owner (None if missing)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
|
||||
|
||||
Used so re-sharing the same conversation in the same mode reuses its
|
||||
existing code instead of minting a new one each time.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
"""List all share links created by ``owner_user_id`` (newest first)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
"""Revoke a share link. Returns False if missing or not owned."""
|
||||
raise NotImplementedError
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
|
||||
|
||||
class MemoryThreadShareStore(ThreadShareStore):
|
||||
def __init__(self) -> None:
|
||||
self._shares: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
code = generate_share_code()
|
||||
while code in self._shares:
|
||||
code = generate_share_code()
|
||||
record = {
|
||||
"share_code": code,
|
||||
"owner_user_id": owner_user_id,
|
||||
"thread_id": thread_id,
|
||||
"revoked": False,
|
||||
"mode": mode,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._shares[code] = record
|
||||
return copy.deepcopy(record)
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
record = self._shares.get(share_code)
|
||||
return copy.deepcopy(record) if record is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
matches = [
|
||||
r
|
||||
for r in self._shares.values()
|
||||
if r["owner_user_id"] == owner_user_id
|
||||
and r["thread_id"] == thread_id
|
||||
and r.get("mode", "import") == mode
|
||||
and not r["revoked"]
|
||||
]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
|
||||
rows.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
record = self._shares.get(share_code)
|
||||
if record is None or record["owner_user_id"] != owner_user_id:
|
||||
return False
|
||||
record["revoked"] = True
|
||||
return True
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""ORM model for conversation share links.
|
||||
|
||||
A share link is a short, revocable code created by a user for *one* of their
|
||||
conversations. Redeeming it copies that single conversation into the redeemer's
|
||||
account. The table only stores the link itself — the conversation is copied
|
||||
lazily at import time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ThreadShareRow(Base):
|
||||
__tablename__ = "thread_shares"
|
||||
|
||||
# Short, URL-safe, unguessable code embedded in the share link.
|
||||
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# The user who created the share.
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# The single conversation (thread) this link shares.
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Soft-disable: a revoked link can no longer be previewed or imported.
|
||||
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
# Share mode:
|
||||
# "import" — recipient (a logged-in user) copies the conversation into
|
||||
# their own account and can keep chatting in the copy.
|
||||
# "view" — a public, read-only static page anyone can open without
|
||||
# logging in; viewers cannot send new messages.
|
||||
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""SQLAlchemy-backed per-conversation share-link repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
|
||||
|
||||
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
created = data.get("created_at")
|
||||
if isinstance(created, datetime):
|
||||
data["created_at"] = created.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class ThreadShareRepository(ThreadShareStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
# Retry on the (astronomically unlikely) code collision.
|
||||
async with self._sf() as session:
|
||||
for _ in range(8):
|
||||
code = generate_share_code()
|
||||
if await session.get(ThreadShareRow, code) is not None:
|
||||
continue
|
||||
row = ThreadShareRow(
|
||||
share_code=code,
|
||||
owner_user_id=owner_user_id,
|
||||
thread_id=thread_id,
|
||||
revoked=False,
|
||||
mode=mode,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
raise RuntimeError("Failed to allocate a unique share code")
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
stmt = (
|
||||
select(ThreadShareRow)
|
||||
.where(
|
||||
ThreadShareRow.owner_user_id == owner_user_id,
|
||||
ThreadShareRow.thread_id == thread_id,
|
||||
ThreadShareRow.mode == mode,
|
||||
ThreadShareRow.revoked.is_(False),
|
||||
)
|
||||
.order_by(ThreadShareRow.created_at.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
if row is None or row.owner_user_id != owner_user_id:
|
||||
return False
|
||||
row.revoked = True
|
||||
await session.commit()
|
||||
return True
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Portable SQLAlchemy column types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
|
||||
# fixed offset is exact and avoids depending on the `tzdata` package (which the
|
||||
# stdlib `zoneinfo` needs on Windows / minimal containers).
|
||||
BEIJING_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
class PortableJSON(TypeDecorator):
|
||||
"""Use native JSON where available, and TEXT on older MySQL servers."""
|
||||
|
||||
impl = JSON
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(Text())
|
||||
return dialect.type_descriptor(JSON())
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None:
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None or not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class BeijingDateTime(TypeDecorator):
|
||||
"""Store timestamps as Beijing (UTC+8) wall-clock time.
|
||||
|
||||
The application works in UTC internally (``datetime.now(UTC)`` everywhere).
|
||||
MySQL ``DATETIME`` and SQLite have no timezone, so the driver serializes the
|
||||
wall-clock fields of whatever datetime it is given and drops ``tzinfo``.
|
||||
Without conversion a row stores the UTC wall clock and an operator browsing
|
||||
the table sees a time 8 hours behind Beijing.
|
||||
|
||||
This decorator converts UTC-aware values to Beijing on the way in (stored as
|
||||
naive Beijing wall-clock) and re-attaches the Beijing zone on the way out, so
|
||||
the database holds Beijing time while Python keeps receiving correct
|
||||
timezone-aware datetimes — instant comparisons stay correct either way.
|
||||
|
||||
PostgreSQL has a real ``TIMESTAMP WITH TIME ZONE`` type and stores the
|
||||
instant unambiguously, so values pass through unchanged there.
|
||||
"""
|
||||
|
||||
impl = DateTime
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
# timezone=True keeps PostgreSQL on timestamptz; MySQL/SQLite ignore it.
|
||||
return dialect.type_descriptor(DateTime(timezone=True))
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Naive values follow the codebase convention of being UTC.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Stored as naive Beijing wall-clock; re-attach the zone.
|
||||
return value.replace(tzinfo=BEIJING_TZ)
|
||||
return value.astimezone(BEIJING_TZ)
|
||||
+4
-3
@@ -13,10 +13,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy import Boolean, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
@@ -25,7 +26,7 @@ class UserRow(Base):
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
@@ -33,7 +34,7 @@ class UserRow(Base):
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
+1
@@ -87,6 +87,7 @@ class RunEventStore(abc.ABC):
|
||||
limit: int = 50,
|
||||
before_seq: int | None = None,
|
||||
after_seq: int | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Return displayable messages (category=message) for a specific run, ordered by seq ascending.
|
||||
|
||||
|
||||
+3
-1
@@ -152,7 +152,9 @@ class JsonlRunEventStore(RunEventStore):
|
||||
events = [e for e in events if e.get("event_type") in event_types]
|
||||
return events[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# JSONL events carry no per-event owner; user_id is accepted for
|
||||
# interface parity with DbRunEventStore but ignored.
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
filtered = [e for e in events if e.get("category") == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
+3
-1
@@ -97,7 +97,9 @@ class MemoryRunEventStore(RunEventStore):
|
||||
filtered = [e for e in filtered if e["event_type"] in event_types]
|
||||
return filtered[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# The in-memory store keeps no per-event owner, so user_id is accepted
|
||||
# for interface parity with DbRunEventStore but ignored.
|
||||
all_events = self._events.get(thread_id, [])
|
||||
filtered = [e for e in all_events if e["run_id"] == run_id and e["category"] == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from .registry import get_scheduled_task_service, set_scheduled_task_service
|
||||
from .service import ScheduledTaskService
|
||||
from .service import DuplicateTaskNameError, ScheduledTaskService
|
||||
|
||||
__all__ = ["ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
|
||||
__all__ = ["DuplicateTaskNameError", "ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
|
||||
+313
-57
@@ -9,7 +9,7 @@ import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -27,6 +27,27 @@ from deerflow.utils.time import now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A single execution attempt may not exceed this wall-clock budget. When it
|
||||
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
|
||||
# and retried. The watchdog uses the same window to recover runs orphaned by a
|
||||
# crash or restart — those stay "running" in the DB with no in-process owner.
|
||||
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
|
||||
# Retries allowed after the first attempt before the run is marked "failed".
|
||||
_MAX_RUN_RETRIES = 3
|
||||
|
||||
|
||||
class DuplicateTaskNameError(Exception):
|
||||
"""A user already owns a scheduled task with the requested name.
|
||||
|
||||
Task names are unique per user (``uq_scheduled_tasks_user_name``). The
|
||||
service raises this instead of letting the raw DB ``IntegrityError`` bubble
|
||||
up so the API layer can return a friendly 409 rather than a 500.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
super().__init__(f"A scheduled task named {name!r} already exists")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RuntimeUser:
|
||||
@@ -337,15 +358,20 @@ class ScheduledTaskService:
|
||||
"kind": item.get("kind"),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _remove_run_archive(run_id: str) -> None:
|
||||
"""Best-effort cleanup of a run's on-disk result archive."""
|
||||
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
|
||||
if archive_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
|
||||
async def delete_run(self, user_id: str, run_id: str) -> bool:
|
||||
deleted = await self.store.delete_run(run_id, user_id)
|
||||
if deleted:
|
||||
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
|
||||
if archive_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
self._remove_run_archive(run_id)
|
||||
return True
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
|
||||
@@ -358,6 +384,20 @@ class ScheduledTaskService:
|
||||
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
return await self.store.update_delivery_profile(user_id, data)
|
||||
|
||||
async def _assert_name_available(self, user_id: str, name: str, *, exclude_task_id: str | None = None) -> None:
|
||||
"""Raise :class:`DuplicateTaskNameError` if ``user_id`` already owns ``name``.
|
||||
|
||||
Enforces ``uq_scheduled_tasks_user_name`` proactively so callers get a
|
||||
clean conflict error instead of a raw DB ``IntegrityError``. The check
|
||||
compares the trimmed name; ``exclude_task_id`` skips the task being
|
||||
renamed so an unchanged name on update is not flagged as a collision.
|
||||
"""
|
||||
for existing in await self.store.list_tasks(user_id):
|
||||
if exclude_task_id is not None and existing.get("task_id") == exclude_task_id:
|
||||
continue
|
||||
if (existing.get("name") or "").strip() == name:
|
||||
raise DuplicateTaskNameError(name)
|
||||
|
||||
async def create_task(
|
||||
self,
|
||||
*,
|
||||
@@ -374,6 +414,8 @@ class ScheduledTaskService:
|
||||
tz = _normalize_timezone(timezone)
|
||||
cron_expr = _schedule_to_cron(schedule)
|
||||
next_run_at = _next_run_at(cron_expr, tz)
|
||||
final_name = name.strip() or "Scheduled task"
|
||||
await self._assert_name_available(user_id, final_name)
|
||||
scheduler_thread = await self.ensure_scheduler_thread(user_id)
|
||||
context = dict(execution_context or {})
|
||||
if execution_agent_name:
|
||||
@@ -386,7 +428,7 @@ class ScheduledTaskService:
|
||||
"source_thread_id": source_thread_id,
|
||||
"source_agent_name": source_agent_name,
|
||||
"execution_agent_name": execution_agent_name,
|
||||
"name": name.strip() or "Scheduled task",
|
||||
"name": final_name,
|
||||
"prompt": prompt.strip(),
|
||||
"schedule_text": schedule.strip(),
|
||||
"cron_expr": cron_expr,
|
||||
@@ -398,31 +440,106 @@ class ScheduledTaskService:
|
||||
)
|
||||
return task
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
@staticmethod
|
||||
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Translate an API update payload into a storage update dict.
|
||||
|
||||
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
|
||||
so both apply identical schedule/timezone normalization.
|
||||
"""
|
||||
update_data: dict[str, Any] = {}
|
||||
for key in ("name", "prompt", "enabled", "execution_agent_name"):
|
||||
if key in data and data[key] is not None:
|
||||
update_data[key] = data[key]
|
||||
# Names are matched against the per-user uniqueness constraint, so keep
|
||||
# the stored value trimmed to match create_task's normalization.
|
||||
if isinstance(update_data.get("name"), str):
|
||||
update_data["name"] = update_data["name"].strip() or "Scheduled task"
|
||||
timezone = data.get("timezone")
|
||||
if timezone is not None:
|
||||
update_data["timezone"] = _normalize_timezone(timezone)
|
||||
if data.get("schedule") is not None:
|
||||
schedule = str(data["schedule"])
|
||||
cron_expr = _schedule_to_cron(schedule)
|
||||
tz = update_data.get("timezone")
|
||||
if tz is None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
tz = (existing or {}).get("timezone", "UTC")
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["schedule_text"] = schedule
|
||||
update_data["cron_expr"] = cron_expr
|
||||
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
|
||||
if data.get("execution_context") is not None:
|
||||
update_data["execution_context"] = data["execution_context"]
|
||||
# Re-enabling a task must also schedule it forward from now, otherwise a
|
||||
# paused task with a stale/empty next_run_at would never fire again.
|
||||
if update_data.get("enabled") is True and "next_run_at" not in update_data:
|
||||
cron = (existing or {}).get("cron_expr")
|
||||
if cron:
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["next_run_at"] = _next_run_at(cron, tz)
|
||||
return update_data
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
if "name" in update_data:
|
||||
await self._assert_name_available(user_id, update_data["name"], exclude_task_id=task_id)
|
||||
return await self.store.update_task(task_id, user_id, update_data)
|
||||
|
||||
async def delete_task(self, user_id: str, task_id: str) -> bool:
|
||||
return await self.store.delete_task(task_id, user_id)
|
||||
|
||||
# --- Admin (cross-user management) ---
|
||||
|
||||
async def admin_list_tasks(self) -> list[dict[str, Any]]:
|
||||
"""List every scheduled task across all users. Admin-only callers."""
|
||||
tasks = await self.store.list_all_tasks()
|
||||
for task in tasks:
|
||||
task["role"] = "owner"
|
||||
return tasks
|
||||
|
||||
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update any task regardless of owner. Admin-only callers."""
|
||||
existing = await self.store.get_task_any(task_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
if "name" in update_data:
|
||||
await self._assert_name_available(existing["user_id"], update_data["name"], exclude_task_id=task_id)
|
||||
return await self.store.update_task_any(task_id, update_data)
|
||||
|
||||
async def admin_delete_task(self, task_id: str) -> bool:
|
||||
"""Delete any task regardless of owner. Admin-only callers."""
|
||||
return await self.store.delete_task_any(task_id)
|
||||
|
||||
async def admin_delete_run(self, run_id: str) -> bool:
|
||||
"""Delete a single run regardless of owner. Admin-only callers."""
|
||||
deleted = await self.store.delete_run_any(run_id)
|
||||
if deleted:
|
||||
self._remove_run_archive(run_id)
|
||||
return deleted
|
||||
|
||||
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""List runs for any task regardless of owner. Admin-only callers."""
|
||||
runs = await self.store.list_runs_any(task_id, limit=limit)
|
||||
for run in runs:
|
||||
run["can_edit"] = True
|
||||
return runs
|
||||
|
||||
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch a single run regardless of owner. Admin-only callers."""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is not None:
|
||||
run["can_edit"] = True
|
||||
return run
|
||||
|
||||
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Trigger an immediate run of any task regardless of owner."""
|
||||
task = await self.store.get_task_any(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
|
||||
return await self.store.update_task(task_id, user_id, {"enabled": False})
|
||||
|
||||
@@ -437,11 +554,19 @@ class ScheduledTaskService:
|
||||
task = await self.store.get_task(task_id, user_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
# Recover hung / orphaned runs first so a single stuck task cannot
|
||||
# silently wedge the scheduler indefinitely.
|
||||
try:
|
||||
await self._reconcile_stuck_runs()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Scheduled task watchdog scan failed")
|
||||
try:
|
||||
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
|
||||
for task in due:
|
||||
@@ -466,64 +591,195 @@ class ScheduledTaskService:
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
"""Run a task while marking it busy so the watchdog won't double-drive it."""
|
||||
self._running_task_ids.add(task["task_id"])
|
||||
try:
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _reconcile_stuck_runs(self) -> None:
|
||||
"""Recover runs left in the ``running`` state with no in-process owner.
|
||||
|
||||
A run becomes stuck when the agent loop hangs past the per-attempt
|
||||
timeout, or when the process crashes / restarts mid-run (the original
|
||||
asyncio task is gone but the DB row still says ``running``). For each
|
||||
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
|
||||
``failed`` so it no longer shows as perpetually executing.
|
||||
"""
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
|
||||
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
|
||||
for run in stuck:
|
||||
task_id = run["task_id"]
|
||||
# A run still owned by a live in-process execution is handled by
|
||||
# that execution's own timeout/retry loop — don't touch it here.
|
||||
if task_id in self._running_task_ids:
|
||||
continue
|
||||
self._running_task_ids.add(task_id)
|
||||
asyncio.create_task(self._recover_and_untrack(run))
|
||||
|
||||
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._recover_stuck_run(run)
|
||||
except Exception:
|
||||
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
|
||||
finally:
|
||||
self._running_task_ids.discard(run["task_id"])
|
||||
|
||||
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
|
||||
run_id = run["id"]
|
||||
retry_count = int(run.get("retry_count") or 0)
|
||||
task = await self.store.get_task_any(run["task_id"])
|
||||
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
|
||||
if task is None:
|
||||
# Parent task was deleted — just close out the orphaned run.
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
|
||||
)
|
||||
return
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
|
||||
|
||||
async def _fail_run(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
run_id: str,
|
||||
scheduled_for: datetime,
|
||||
*,
|
||||
error: str,
|
||||
retry_count: int = 0,
|
||||
) -> None:
|
||||
"""Mark a run failed and reflect the outcome on the parent task."""
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
|
||||
async def _execute_task(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
scheduled_for: datetime,
|
||||
manual: bool,
|
||||
run_row: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
# Each run uses its own throwaway thread so message history cannot accumulate
|
||||
# across runs and blow past the model's input window. The per-user
|
||||
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
|
||||
# we keep it on the task row only for backwards compatibility.
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": runtime_thread_id,
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
is_recovery = run_row is not None
|
||||
if run_row is None:
|
||||
return
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": str(uuid.uuid4()),
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
if run_row is None:
|
||||
return
|
||||
else:
|
||||
# Recovery path: re-claim an existing (orphaned) run row.
|
||||
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
|
||||
|
||||
if not manual:
|
||||
# Advancing the schedule only happens once, when the run is first
|
||||
# created — recovery retries reuse the original scheduled_for.
|
||||
if not manual and not is_recovery:
|
||||
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
|
||||
|
||||
try:
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_row["id"],
|
||||
output_snapshot=output_snapshot,
|
||||
run_id = run_row["id"]
|
||||
retry_count = int(run_row.get("retry_count") or 0)
|
||||
|
||||
while True:
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
|
||||
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
except TimeoutError:
|
||||
# The attempt overran the 30-minute cap. Retry a bounded number
|
||||
# of times, then give up and mark the run failed.
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
retry_count += 1
|
||||
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{
|
||||
"retry_count": retry_count,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
|
||||
},
|
||||
)
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
|
||||
return
|
||||
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
|
||||
run_id,
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
|
||||
await self._deliver_element_result(task, result["content"])
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
result = _build_result(
|
||||
task,
|
||||
scheduled_for=scheduled_for,
|
||||
content=f"Scheduled task failed: {exc}",
|
||||
return
|
||||
|
||||
async def _run_agent_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the agent once and build the result payload.
|
||||
|
||||
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
|
||||
cannot hold the shared run lock — and the whole scheduler — forever.
|
||||
"""
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_id,
|
||||
output_snapshot=output_snapshot,
|
||||
)
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
|
||||
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
|
||||
token = set_current_user(_RuntimeUser(id=user_id))
|
||||
|
||||
@@ -41,13 +41,14 @@ dependencies = [
|
||||
[project.optional-dependencies]
|
||||
hindsight = ["hindsight-client>=0.4.22"]
|
||||
ollama = ["langchain-ollama>=0.3.0"]
|
||||
postgres = [
|
||||
"asyncpg>=0.29",
|
||||
"langgraph-checkpoint-postgres>=3.0.5",
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
postgres = [
|
||||
"asyncpg>=0.29",
|
||||
"langgraph-checkpoint-postgres>=3.0.5",
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
mysql = ["asyncmy>=0.2.10"]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
|
||||
Reference in new issue
Block a user