Merge remote-tracking branch 'origin/master' into xuhd

# Conflicts:
#	offline-backend-20260512/backend/config.yaml
This commit is contained in:
xuhd committed 2026-06-09 15:43:40 +08:00
commit bd0eceda61
267 files changed
+41209 -14541

No files matched your search

@@ -1,8 +1,10 @@
"""Middleware to inject uploaded files information into agent context."""
import logging
from pathlib import Path
from typing import NotRequired, override
import logging
import shutil
from pathlib import Path
from typing import NotRequired, override
from urllib.parse import unquote
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
@@ -13,7 +15,7 @@ from deerflow.config.paths import Paths, get_paths
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.file_conversion import extract_outline
logger = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
_OUTLINE_PREVIEW_LINES = 5
@@ -22,7 +24,8 @@ _OUTLINE_PREVIEW_LINES = 5
# configured vision model), never by reading their bytes. Listing them with the
# generic read_file/grep guidance made the agent try to parse image binaries
# with bash/python, which fails and wastes turns.
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
_ARTIFACT_URL_MARKER = "/artifacts/"
def _extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]:
@@ -69,7 +72,7 @@ class UploadsMiddlewareState(AgentState):
uploaded_files: NotRequired[list[dict] | None]
class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
"""Middleware to inject uploaded files information into the agent context.
Reads file metadata from the current message's additional_kwargs.files
@@ -122,7 +125,7 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
lines.append(" Use `grep` to search for keywords (e.g. `grep(pattern='keyword', path='/mnt/user-data/uploads/')`).")
lines.append("")
def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str:
def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str:
"""Create a formatted message listing uploaded files.
Args:
@@ -160,9 +163,61 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
lines.append("- Only fall back to web search if the file content is clearly insufficient to answer the question.")
lines.append("</uploaded_files>")
return "\n".join(lines)
def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None:
return "\n".join(lines)
def _import_file_from_artifact_url(
self,
file_info: dict,
uploads_dir: Path | None,
) -> bool:
"""Copy a previously uploaded file into the current thread when possible.
iframe-style callers may pass file metadata that points at an upload
from another DeerFlow thread. The current run thread still needs a real
file under its own ``/mnt/user-data/uploads`` directory, otherwise this
middleware will filter the entry out during existence checks.
"""
filename = file_info.get("filename") or ""
artifact_url = str(file_info.get("artifact_url") or "").strip()
if not filename or uploads_dir is None or not artifact_url:
return False
if not artifact_url.startswith("/api/threads/") or _ARTIFACT_URL_MARKER not in artifact_url:
return False
prefix, encoded_virtual_path = artifact_url.split(_ARTIFACT_URL_MARKER, 1)
source_thread_id = prefix.removeprefix("/api/threads/").strip("/")
if not source_thread_id or not encoded_virtual_path:
return False
virtual_path = "/" + unquote(encoded_virtual_path.lstrip("/"))
if Path(virtual_path).name != filename:
return False
try:
source_path = self._paths.resolve_virtual_path(
source_thread_id,
virtual_path,
user_id=get_effective_user_id(),
)
except ValueError:
logger.debug("Failed to resolve uploaded artifact path: %s", artifact_url, exc_info=True)
return False
if not source_path.is_file():
return False
uploads_dir.mkdir(parents=True, exist_ok=True)
target_path = uploads_dir / filename
shutil.copy2(source_path, target_path)
source_markdown = source_path.with_suffix(".md")
if source_markdown.is_file() and source_markdown != source_path:
shutil.copy2(source_markdown, target_path.with_suffix(".md"))
return True
def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None:
"""Extract file info from message additional_kwargs.files.
The frontend sends uploaded file metadata in additional_kwargs.files
@@ -183,16 +238,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
files = []
for f in kwargs_files:
if not isinstance(f, dict):
continue
filename = f.get("filename") or ""
if not filename or Path(filename).name != filename:
continue
if uploads_dir is not None and not (uploads_dir / filename).is_file():
continue
files.append(
{
"filename": filename,
if not isinstance(f, dict):
continue
filename = f.get("filename") or ""
if not filename or Path(filename).name != filename:
continue
if uploads_dir is not None and not (uploads_dir / filename).is_file():
if not self._import_file_from_artifact_url(f, uploads_dir):
continue
files.append(
{
"filename": filename,
"size": int(f.get("size") or 0),
"path": f"/mnt/user-data/uploads/{filename}",
"extension": Path(filename).suffix,
@@ -243,6 +299,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
# Get newly uploaded files from the current message's additional_kwargs.files
new_files = self._files_from_kwargs(last_message, uploads_dir) or []
# Only inject when the *current* message actually carries new uploads.
# Turns without a fresh upload must not re-inject previously uploaded
# files: the original upload turn already injected (and persisted) their
# <uploaded_files> block, and the files remain physically available for
# read_file/grep/view_image. Re-injecting on every later message made the
# agent re-reference old images/docs, and — since the frontend parses
# files out of the <uploaded_files> content when additional_kwargs.files
# is empty — re-rendered the old attachment on each new question.
if not new_files:
return None
# Collect historical files from the uploads directory (all except the new ones)
new_filenames = {f["filename"] for f in new_files}
historical_files: list[dict] = []
@@ -270,9 +337,6 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
file["outline"] = outline
file["outline_preview"] = preview
if not new_files and not historical_files:
return None
logger.debug(f"New files: {[f['filename'] for f in new_files]}, historical: {[f['filename'] for f in historical_files]}")
# Create files message and prepend to the last human message content
@@ -47,6 +47,7 @@ class OrchestrationResult:
pending_clarification: str | None = None
error: str | None = None
last_leader_content: str = ""
dialogues: list = field(default_factory=list)
@dataclass
@@ -71,6 +72,7 @@ async def run_orchestration(
seed_message: str,
mode: str = "recommend",
intent_text: str = "",
enable_report: bool = True,
) -> OrchestrationResult:
"""运行一次完整编排(循环 → 共识 → 报告绘制)。
@@ -84,13 +86,17 @@ async def run_orchestration(
seed_message: 第一轮 leader 的 prompt(启动=初始意图;resume=用户答案续轮)。
mode: "recommend" | "chain"。
intent_text: 任务意图,用于报告 prompt(缺省退回 seed_message)。
enable_report: 是否在收口后执行 Step3 报告绘制。False 时只跑完 Step2 研讨即收尾、
不生成报告(step3=None)。当前「后台挂起」临时禁用报告(进程内驱动报告 agent
在真模型下仍无法稳定写出 HTML,详见 multi-agent-background-run-遗留问题与下次继续.md)。
"""
intent = (intent_text or seed_message or "").strip()
try:
await progress.begin()
if mode == "chain":
outcome = await _run_chain(gateway, progress, agents, thread_ids, coordinator_name, model, seed_message)
outcome = await _run_chain(gateway, progress, agents, thread_ids, coordinator_name, model, seed_message, intent)
else:
outcome = await _run_recommend(gateway, progress, agents, thread_ids, coordinator_name, model, seed_message)
outcome = await _run_recommend(gateway, progress, agents, thread_ids, coordinator_name, model, seed_message, intent)
if outcome.kind == "clarification":
question = outcome.question or "总控需要补充信息,请补充后继续。"
@@ -99,33 +105,74 @@ async def run_orchestration(
status="awaiting_input",
pending_clarification=question,
last_leader_content=outcome.last_leader_content,
dialogues=progress.dialogues(),
)
# consensus 或 max_cycles:均收口 → 报告绘制
# consensus 或 max_cycles:均收口。报告绘制按 enable_report 开关执行。
await progress.consensus(100)
step3 = await _run_report(
gateway,
progress,
agents=agents,
model=model,
intent_text=intent_text or seed_message,
consensus_text=outcome.last_leader_content,
deliveries=outcome.deliveries,
)
step3 = None
if enable_report:
step3 = await _run_report(
gateway,
progress,
agents=agents,
model=model,
intent_text=intent_text or seed_message,
consensus_text=outcome.last_leader_content,
deliveries=outcome.deliveries,
)
await progress.done()
return OrchestrationResult(
status="done",
consensus=True,
step3=step3,
last_leader_content=outcome.last_leader_content,
dialogues=progress.dialogues(),
)
except Exception as exc: # noqa: BLE001 — 顶层兜底:标记 error 并返回
logger.exception("roundtable orchestration failed")
await progress.error(str(exc))
return OrchestrationResult(status="error", error=str(exc))
return OrchestrationResult(status="error", error=str(exc), dialogues=progress.dialogues())
# ── recommend 模式 ───────────────────────────────────────────────────────────
# ── 上下文「广播」(inline 实现)────────────────────────────────────────────────
#
# 进程内网关没有 multi_agent 的 thread 广播,席位/总控会看不到「原始任务背景 + 各席位
# 前序交付」→ 席位反问用户、总控靠记忆瞎编。这里把这些信息**直接拼进 prompt**:
# - 每个席位的任务带上:任务背景 + 前序各席位交付 + 「直接产出、别提问」硬约束;
# - 总控的续轮 prompt 带上:各席位本轮交付,让它据实派活/收口。
_SEAT_NO_ASK = (
"注意:这是自动化圆桌研讨,没有用户在线应答——**请勿向用户提问或等待确认**,"
"直接基于已知信息做出合理假设,产出完整的结构化交付。\n"
"**交付方式(与前端手动研讨一致)**:请用 write_file 工具把你这一轮的完整交付整理成一份 "
"Markdown 文件,写入 /mnt/user-data/outputs/ 下(文件名体现你的职责,如 `情报收集.md`、"
"`风险评估.md`);同时在回复正文里给出这份交付的内容本身(供总控汇总),"
"不要只回复『已写入文件』这类空话。"
)
def _deliveries_block(deliveries: dict[str, str], name_by_id: dict[str, str]) -> str:
parts: list[str] = []
for aid, text in deliveries.items():
t = (text or "").strip()
if not t:
continue
parts.append(f"### {name_by_id.get(aid, aid)}\n{t}")
return "\n\n".join(parts)
def _seat_message(intent_text: str, deliveries: dict[str, str], name_by_id: dict[str, str], task: str) -> str:
"""把任务背景 + 前序交付 + 防提问约束拼进席位任务(替代 thread 广播)。"""
parts: list[str] = []
if intent_text.strip():
parts.append(f"【本次研讨任务背景】\n{intent_text.strip()}")
block = _deliveries_block(deliveries, name_by_id)
if block:
parts.append("【前序各席位已完成的交付,请在此基础上承接、补充、不要重复或推翻】\n" + block)
parts.append(f"【你的本轮具体任务】\n{task}")
parts.append(_SEAT_NO_ASK)
return "\n\n---\n\n".join(parts)
async def _run_recommend(
@@ -136,10 +183,12 @@ async def _run_recommend(
coordinator_name: str,
model: str | None,
seed_message: str,
intent_text: str,
) -> _LoopOutcome:
deliveries: dict[str, str] = {}
last_leader = ""
message = seed_message
name_by_id = {a.agent_id: a.name for a in agents}
for cycle in range(1, MAX_CYCLES + 1):
await progress.leader_thinking(cycle)
@@ -151,22 +200,32 @@ async def _run_recommend(
)
if turn.final_text:
last_leader = turn.final_text
await progress.add_leader_dialogue(
turn.final_text, cycle, tool_calls=turn.tool_calls, thread_id=thread_ids.get(coordinator_name)
)
if turn.is_clarification:
return _LoopOutcome("clarification", question=turn.clarification, last_leader_content=last_leader, deliveries=deliveries)
if turn.is_consensus:
return _LoopOutcome("consensus", last_leader_content=last_leader, deliveries=deliveries)
# **顺序**派活 —— 不要并行
# **顺序**派活 —— 不要并行;每个席位带上「背景 + 前序交付」上下文
await progress.dispatching(cycle)
for agent_id, task in turn.dispatched:
await progress.seat_active(agent_id)
seat = await gateway.run_seat(thread_ids=thread_ids, agent_id=agent_id, task=task, model=model)
seat_msg = _seat_message(intent_text, deliveries, name_by_id, task)
seat = await gateway.run_seat(thread_ids=thread_ids, agent_id=agent_id, task=seat_msg, model=model)
deliveries[agent_id] = seat.final_text
await progress.add_seat_dialogue(
agent_id, name_by_id.get(agent_id, agent_id), seat.final_text, cycle,
tool_calls=seat.tool_calls, thread_id=thread_ids.get(agent_id),
)
await progress.seat_done(agent_id, cycle)
await progress.bump_consensus()
message = CONTINUE_PROMPT
# 续轮:把各席位本轮交付喂给总控,让它据实判断(总控 SOUL 依赖看到交付)
block = _deliveries_block(deliveries, name_by_id)
message = CONTINUE_PROMPT + (f"\n\n各席位最新交付如下:\n{block}" if block else "")
# 达到最大轮次:强制收口(对齐前端 toast「已达到最大研讨轮次」)
logger.info("roundtable recommend loop hit MAX_CYCLES=%d, forcing consensus", MAX_CYCLES)
@@ -184,13 +243,15 @@ async def _run_chain(
coordinator_name: str,
model: str | None,
seed_message: str,
intent_text: str,
) -> _LoopOutcome:
deliveries: dict[str, str] = {}
last_leader = ""
total = len(agents)
name_by_id = {a.agent_id: a.name for a in agents}
for i, seat in enumerate(agents):
# (a) 总控按链条顺序只给当前席位派活
# (a) 总控按链条顺序只给当前席位派活;带上前序各席位交付,让它据实派活
await progress.leader_thinking(i + 1)
leader_task = build_chain_leader_task(
seed_message=seed_message,
@@ -200,6 +261,9 @@ async def _run_chain(
seat_name=seat.name,
is_first=(i == 0),
)
block = _deliveries_block(deliveries, name_by_id)
if block:
leader_task += f"\n\n前序各席位已完成的交付如下,请据此为本席位分配承接性任务:\n{block}"
turn = await gateway.run_leader(
thread_ids=thread_ids,
coordinator_name=coordinator_name,
@@ -208,31 +272,44 @@ async def _run_chain(
)
if turn.final_text:
last_leader = turn.final_text
await progress.add_leader_dialogue(
turn.final_text, i + 1, tool_calls=turn.tool_calls, thread_id=thread_ids.get(coordinator_name)
)
if turn.is_clarification:
return _LoopOutcome("clarification", question=turn.clarification, last_leader_content=last_leader, deliveries=deliveries)
task = pick_chain_task(turn.dispatched, turn.final_text, seat.agent_id, seat.name)
# (b) 当前席位交付(顺序,不并行)
# (b) 当前席位交付(顺序,不并行);带上「背景 + 前序交付 + 别提问」上下文
await progress.seat_active(seat.agent_id)
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=task, model=model)
seat_msg = _seat_message(intent_text, deliveries, name_by_id, task)
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=seat_msg, model=model)
deliveries[seat.agent_id] = seat_turn.final_text
await progress.add_seat_dialogue(
seat.agent_id, seat.name, seat_turn.final_text, i + 1,
tool_calls=seat_turn.tool_calls, thread_id=thread_ids.get(seat.agent_id),
)
await progress.seat_done(seat.agent_id, i + 1)
if total > 0:
await progress.set_consensus_pct(round((i + 1) / total * 90))
# (c) 总控收口:综合全链交付,产出最终结论
# (c) 总控收口:综合全链交付,产出最终结论(带上全部席位交付)
if total > 0:
await progress.leader_thinking(total + 1)
block = _deliveries_block(deliveries, name_by_id)
closeout_msg = CHAIN_CLOSEOUT_PROMPT + (f"\n\n各席位完整交付如下:\n{block}" if block else "")
closeout = await gateway.run_leader(
thread_ids=thread_ids,
coordinator_name=coordinator_name,
message=CHAIN_CLOSEOUT_PROMPT,
message=closeout_msg,
model=model,
)
if closeout.final_text:
last_leader = closeout.final_text
await progress.add_leader_dialogue(
closeout.final_text, total + 1, tool_calls=closeout.tool_calls, thread_id=thread_ids.get(coordinator_name)
)
return _LoopOutcome("consensus", last_leader_content=last_leader, deliveries=deliveries)
@@ -273,27 +350,43 @@ def build_report_prompt(
席位交付按 ``agents`` 顺序列出(缺交付的席位跳过)。报告智能体据此用 write_file
写出自包含的 ``outputs/方案总览.html``(具体绘制规则在 roundtable-report 的 SOUL)。
"""
# 与前端 useStep3Report.buildReportPrompt 对齐 —— 关键是**明确叫报告 agent 用
# write_file 写到 /mnt/user-data/outputs/方案总览.html**,否则它只会在对话里回文字、
# 不落 HTML 文件(_read_report_html 就读不到)。
name_by_id = {a.agent_id: a.name for a in agents}
lines: list[str] = [
"【圆桌最终成果汇总 —— 请据此绘制方案总览 HTML 看板】",
"请基于以下圆桌多智能体研讨的成果,撰写一份图文并茂、以文字叙述为主线的 HTML「方案研讨汇报」,"
"用 write_file 工具写入 /mnt/user-data/outputs/方案总览.html。",
"目标:让用户能轻松读懂这群智能体讨论出了什么方案、为什么是它。请逐个席位提炼其核心观点与交付写成文字,"
"再综合成最终方案;图表用来点睛(风险矩阵、时间线、资源占比、贡献雷达等),详略得当——"
"缺明确数字时也要据研讨内容合理估算出图并标注「定性/估算」,不要通篇只有文字。",
"",
"## 任务意图",
"# 一、任务意图",
(intent_text or "").strip() or "(未提供)",
"",
"## 各席位交付",
"# 二、各席位最终交付",
]
name_by_id = {a.agent_id: a.name for a in agents}
has_delivery = False
for a in agents:
for i, a in enumerate(agents):
text = (deliveries.get(a.agent_id) or "").strip()
if not text:
continue
has_delivery = True
lines.append(f"### {name_by_id.get(a.agent_id, a.agent_id)}")
lines.append(f"## {i + 1}. {name_by_id.get(a.agent_id, a.agent_id)}")
lines.append(text)
lines.append("")
if not has_delivery:
lines.append("(各席位暂无可用交付,请基于任务意图与总控结论绘制。)")
lines.append("(各席位暂无文字交付,请基于任务意图与总控结论合理展开成方案。)")
lines.append("")
lines.append("## 总控最终结论")
lines.append("# 三、总控最终共识结论")
lines.append((consensus_text or "").strip() or "(未提供)")
lines.append("")
lines.append(
"请严格遵循你的职责设定:自包含离线(纯 inline SVG 图表、禁止任何外链 CDN)、研讨叙事型结构"
"(结论+KPI → 背景目标 → 各席位观点 → 综合分析 → 最终共识方案 → 执行路线)、"
"文字为主图表点睛、浅红主题(主色 #c25555);写完文件后只用一两句话总结,不要把 HTML 源码贴进对话。"
)
lines.append(
'硬性要求:① 全程使用简体中文;② HTML 必须是完整文档,<head> 最前面声明 <meta charset="utf-8">。'
)
return "\n".join(lines)
@@ -10,12 +10,21 @@
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Any, Awaitable, Callable
from .types import SeatRef
logger = logging.getLogger(__name__)
# 北京时区(UTC+8,固定无 DST)——对话气泡时间戳用,对齐前端 nowTime()
# = new Date().toLocaleTimeString("zh-CN", {hour12:false}) 的 "HH:MM:SS" 形状。
_BEIJING_TZ = timezone(timedelta(hours=8))
def _now_hms() -> str:
return datetime.now(_BEIJING_TZ).strftime("%H:%M:%S")
# 节点状态机:pending → active → done。
_PENDING = "pending"
_ACTIVE = "active"
@@ -42,14 +51,21 @@ class JobProgressWriter:
job_id: str,
agents: list[SeatRef],
event_sink: EventSink = None,
on_dialogue: Any | None = None,
) -> None:
self._store = store
self._job_id = job_id
self._agents = list(agents)
self._event_sink = event_sink
# 每轮 leader/seat 模型返回后回调(dialogues, consensus)—— 执行器据此把对话
# 写进草稿(和手动跑同一存取路径)。
self._on_dialogue = on_dialogue
self._state: dict[str, str] = {a.agent_id: _PENDING for a in agents}
self._spoke_cycle: dict[str, int] = {}
self._consensus = 0
# 研讨对话记录(leader 说明 + 各席位交付),随每次落库一并写入。
self._dialogues: list[dict[str, Any]] = []
self._dlg_seq = 0
# ── 节点快照 ──────────────────────────────────────────────────────────
@@ -76,6 +92,7 @@ class JobProgressWriter:
"""落库(始终带最新 dispatch_chain + consensus)+ 外发事件,均 best-effort。"""
fields.setdefault("dispatch_chain", self._nodes())
fields.setdefault("consensus_percentage", self._consensus)
fields.setdefault("dialogues", self._dialogues)
if self._store is not None:
try:
await self._store.update_progress(self._job_id, **fields)
@@ -140,3 +157,78 @@ class JobProgressWriter:
async def error(self, message: str) -> None:
await self._persist(status="error", error=message)
# ── 对话记录 ─────────────────────────────────────────────────────────────
def _add(
self,
role: str,
agent_id: str | None,
name: str,
content: str,
cycle: int,
*,
tool_calls: list[dict] | None = None,
thread_id: str | None = None,
) -> None:
self._dlg_seq += 1
self._dialogues.append(
{
"id": self._dlg_seq,
"role": role,
"agentId": agent_id,
"name": name,
"content": content,
"cycle": cycle,
# 工具步骤卡数据源(每项 {name, args, id});映射成前端 Step2Dialogue.steps。
"toolCalls": list(tool_calls or []),
# 该气泡对应的 leader/seat 线程——映射成 agentThreadId,供点开产物 / 反查 stub。
"threadId": thread_id,
# 气泡时间戳,**创建时一次性盖章**(不随后续 flush 抖动),对齐前端 nowTime()。
"time": _now_hms(),
}
)
def dialogues(self) -> list[dict[str, Any]]:
return list(self._dialogues)
async def _flush_dialogue(self) -> None:
if self._on_dialogue is None:
return
try:
await self._on_dialogue(self.dialogues(), self._consensus)
except Exception: # noqa: BLE001 — 写草稿失败不打断编排
logger.warning("roundtable job dialogue → draft flush failed", exc_info=True)
async def add_leader_dialogue(
self,
content: str,
cycle: int,
*,
tool_calls: list[dict] | None = None,
thread_id: str | None = None,
) -> None:
"""总控一轮发言(派活说明 / 共识结论)。空内容跳过。
``tool_calls`` / ``thread_id`` 用于重建工具步骤卡(派活 chip)+ agentThreadId。
"""
if not (content or "").strip():
return
self._add("leader", None, "总控协调", content, cycle, tool_calls=tool_calls, thread_id=thread_id)
await self._persist()
await self._flush_dialogue()
async def add_seat_dialogue(
self,
agent_id: str,
name: str,
content: str,
cycle: int,
*,
tool_calls: list[dict] | None = None,
thread_id: str | None = None,
) -> None:
"""某席位一轮交付。``tool_calls`` / ``thread_id`` 用于步骤卡 + 产物预览。"""
self._add("seat", agent_id, name, content or "", cycle, tool_calls=tool_calls, thread_id=thread_id)
await self._persist()
await self._flush_dialogue()
@@ -52,6 +52,9 @@ class LeaderTurn:
dispatched: list[tuple[str, str]] = field(default_factory=list)
# 命中澄清时的问题文本;非 None 即 clarification 分支。
clarification: str | None = None
# 本轮总控调用的工具(按出现顺序,每项 {name, args, id})——通常是 agent_orchestration
# 派活 / ask_clarification。用于在草稿 run 里重建「工具步骤卡」,与前端手动跑一致。
tool_calls: list[dict] = field(default_factory=list)
@property
def is_clarification(self) -> bool:
@@ -67,6 +70,9 @@ class SeatTurn:
"""某席位一轮交付的归一化结果(由 gateway.run_seat 产出)。"""
final_text: str = ""
# 本轮席位调用的工具(按出现顺序,每项 {name, args, id})——用于在草稿 run 里
# 重建「工具步骤卡」+ write_file stub(点开产物),与前端手动跑一致。
tool_calls: list[dict] = field(default_factory=list)
@dataclass
@@ -0,0 +1,92 @@
"""Server-side mirror of the frontend agent-name desensitization map.
The web UI displays agent names through a static term→abbreviation map
(``frontend-web/src/lib/desensitize.ts``) — e.g. the stored name "态势分析" is
shown as "TS分析". The gallery search box therefore matches against the
*desensitized* display form, not the raw stored name.
When search moves server-side (paginated listing), the backend must match the
same way, so this module mirrors that static map verbatim. The frontend map is
never extended at runtime (``addDesensitizeEntries`` is defined but never
called), so a static copy stays in sync.
Used by the SQL listing query to match "raw name OR desensitized name".
"""
from __future__ import annotations
# Mirror of DESENSITIZE_MAP in frontend-web/src/lib/desensitize.ts.
# Keep entries identical to the frontend so server-side and client-side search
# behave the same. Order does not affect membership tests.
DESENSITIZE_MAP: dict[str, str] = {
"态势": "TS",
"你好": "nh",
"滨海作战体系": "第一张网",
"滨海作战部队": "第一张网",
"Taiwan Air Force Flight Units": "第二张网",
"台海空军飞行部队": "第二张网",
"空军": "KJ",
"海军": "HJ",
"陆军": "LJ",
"陆战队": "LZD",
"海巡": "HX",
"憲兵": "XB",
"資通電軍": "ZTDJ",
"情报": "QB",
"指挥": "ZH",
"信息准备": "XX准备",
"行为体": "XWT",
"驱动因素": "QDYS",
"叙事": "XS",
"前三定": "QSD",
"受众目标": "SZMB",
"作战": "ZZ",
"目的": "MD",
"关键传播节点": "GJCBJD",
"战法": "ZF",
"行动": "XD",
"打击": "DJ",
"目标": "MB",
"战力": "ZL",
"筹划": "COH",
"策划": "CEH",
"模拟": "MN",
"推演": "TY",
"认知": "RZ",
"触及": "CJ",
"军": "J",
"美军": "MJ",
"台军": "TJ",
}
def desensitize_name(name: str) -> str:
"""Return ``name`` with every mapped term replaced by its abbreviation.
Mirrors the frontend ``desensitize()`` — a sequence of literal replacements.
"""
out = name
for term, abbr in DESENSITIZE_MAP.items():
if term in out:
out = out.replace(term, abbr)
return out
def name_matches(name: str, needle: str) -> bool:
"""True when ``needle`` (case-insensitive) is in the raw OR desensitized name."""
n = needle.strip().lower()
if not n:
return True
return n in name.lower() or n in desensitize_name(name).lower()
def source_terms_for_needle(needle: str) -> list[str]:
"""Return the raw source terms whose abbreviation contains ``needle``.
Lets the SQL search add ``name LIKE %term%`` clauses so a user typing an
abbreviation (e.g. "TS") still matches the stored raw term (e.g. "态势").
"""
n = needle.strip().lower()
if not n:
return []
return [term for term, abbr in DESENSITIZE_MAP.items() if n in abbr.lower()]
@@ -15,6 +15,7 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
@@ -108,6 +109,7 @@ class AppConfig(BaseModel):
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
log_system: LogSystemConfig = Field(default_factory=LogSystemConfig, description="Admin-only external log-system entry shown in the settings menu")
sandbox: SandboxConfig = Field(description="Sandbox configuration")
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
@@ -2,7 +2,7 @@
from typing import Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field
CheckpointerType = Literal["memory", "sqlite", "postgres"]
@@ -16,13 +16,41 @@ class CheckpointerConfig(BaseModel):
"'sqlite' persists to a local file (requires langgraph-checkpoint-sqlite). "
"'postgres' persists to PostgreSQL (requires langgraph-checkpoint-postgres)."
)
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
"Required for sqlite and postgres types. "
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
use_postgres: bool = Field(
default=True,
description=(
"When type='postgres', set false to keep the Postgres DSN in config "
"but run the checkpointer on SQLite instead."
),
)
sqlite_connection_string: str | None = Field(
default=None,
description=(
"SQLite file used when type='postgres' and use_postgres=false. "
"Defaults to the first dual_read_sqlite_paths entry or 'store.db'."
),
)
dual_read_enabled: bool = Field(
default=False,
description=(
"When type='postgres', read from Postgres first and fall back to legacy "
"SQLite checkpoint files when Postgres does not have the requested data."
),
)
dual_read_sqlite_paths: list[str] = Field(
default_factory=list,
description=(
"Legacy SQLite checkpoint database paths used for read fallback. "
"Plain paths are resolved the same way as SQLite connection_string."
),
)
# Global configuration instance — None means no checkpointer is configured.
@@ -11,13 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
No database is initialized.
@@ -26,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
to reference environment variables from .env:
database:
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
"""
from __future__ import annotations
@@ -42,40 +42,54 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
echo_sql: bool = Field(
default=False,
description="Echo all SQL statements to log (debug only).",
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"SQLAlchemy default is 30."
),
)
# -- Derived helpers (not user-configured) --
@@ -108,16 +122,16 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
@@ -0,0 +1,18 @@
"""Configuration for the admin-only external log-system entry.
A small read-only section in ``config.yaml`` that lets a deployment point the
"日志系统" entry (shown in the bottom-right settings menu for admins) at an
external log/observability system. The URL is opened in a new browser tab.
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class LogSystemConfig(BaseModel):
"""External log-system link surfaced in the admin settings menu."""
enabled: bool = Field(default=False, description="Whether the log-system entry is shown in the settings menu")
url: str = Field(default="", description="External log-system URL opened in a new tab when the entry is clicked")
label: str = Field(default="日志系统", description="Menu label shown for the entry")
@@ -51,6 +51,14 @@ class AppearanceSettings(BaseModel):
default="light",
description="Theme applied for users who haven't picked their own (light / dark)",
)
default_skill_view: Literal["card", "list"] = Field(
default="list",
description="Default skill list display mode for users who haven't picked their own (card / list)",
)
default_agent_view: Literal["card", "list"] = Field(
default="card",
description="Default agent list display mode for users who haven't picked their own (card / list)",
)
model_config = ConfigDict(extra="ignore")
@@ -78,6 +86,17 @@ class PrivateSquareSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LeaderboardSettings(BaseModel):
"""Admin-configurable defaults for the user leaderboard analytics."""
exclude_user_ids: list[str] = Field(default_factory=list, description="User ids excluded from leaderboard aggregation.")
include_scheduled: bool = Field(default=True, description="Whether scheduled task runs participate in leaderboard statistics.")
include_admins: bool = Field(default=True, description="Whether admin accounts participate in leaderboard statistics.")
include_failed: bool = Field(default=True, description="Whether failed runs and provider calls participate in leaderboard statistics.")
cleanup_system_questions: bool = Field(default=True, description="Hide attachment/system marker questions such as <uploaded_files>.")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -85,6 +104,7 @@ class SystemSettings(BaseModel):
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
model_config = ConfigDict(extra="ignore")
@@ -0,0 +1,5 @@
"""Persistence models for admin analytics snapshots."""
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
__all__ = ["AdminLeaderboardDailyStatRow"]
@@ -0,0 +1,31 @@
"""ORM models for admin analytics snapshots."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AdminLeaderboardDailyStatRow(Base):
"""One precomputed leaderboard snapshot for a single local calendar day."""
__tablename__ = "admin_leaderboard_daily_stats"
__table_args__ = (
Index("ix_admin_leaderboard_daily_stats_status", "status"),
Index("ix_admin_leaderboard_daily_stats_updated_at", "updated_at"),
)
stat_date: Mapped[str] = mapped_column(String(10), primary_key=True)
settings_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
status: Mapped[str] = mapped_column(String(16), default="ready")
payload_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
message: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
generated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -4,13 +4,20 @@ from __future__ import annotations
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.memory import MemoryAgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.agents.sql import AgentRepository
__all__ = [
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRepository",
"AgentRow",
"AgentSkillRow",
"AgentStore",
"MemoryAgentStore",
"make_agent_store",
@@ -118,6 +118,69 @@ class AgentStore(ABC):
"""
raise NotImplementedError
@abstractmethod
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
``scope``:
- ``"mine"`` — owned by ``user_id`` OR favorited by ``user_id``.
- ``"square"`` — built-in or published, and not private-square.
- ``"private"``— flagged ``is_private=True`` (caller gates access).
- ``None`` — the default listing visibility (``list_visible``).
``admin=True`` bypasses the visibility/scope filter entirely (every
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). Both filters,
the COUNT, and LIMIT/OFFSET all execute in SQL.
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
for the 1-based ``page``.
"""
raise NotImplementedError
# --- Listing cache (skills / tool_groups / model) ---
@abstractmethod
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
"""Replace the cached skill list for ``agent_id``.
``None`` and ``[]`` both clear the cache to "no skills" (mirrors the
tri-state semantics of ``AgentConfig.skills``).
"""
raise NotImplementedError
@abstractmethod
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
raise NotImplementedError
@abstractmethod
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
"""Batch-fetch cached listing extras for ``agent_ids``.
Returns ``{agent_id: {"skills": list[str] | None, "tool_groups":
list[str] | None, "model": str | None}}``. An agent that was never
mirrored is absent from the mapping (caller treats it as unknown).
"""
raise NotImplementedError
@abstractmethod
async def delete_extras_for_agent(self, agent_id: str) -> None:
"""Drop the cached skills + extras rows for ``agent_id`` (on delete)."""
raise NotImplementedError
# --- Favorites ("收藏") ---
@abstractmethod
@@ -136,6 +199,28 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
# --- Per-user pins ("置顶") ---
@abstractmethod
async def add_pin(self, user_id: str, agent_id: str) -> None:
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
"""Remove a per-user pin. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_pin_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids pinned by ``user_id``."""
raise NotImplementedError
@abstractmethod
async def delete_pins_for_agent(self, agent_id: str) -> None:
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
@@ -5,6 +5,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
@@ -20,8 +21,12 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
@@ -166,6 +171,96 @@ class MemoryAgentStore(AgentStore):
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
@@ -180,5 +275,23 @@ class MemoryAgentStore(AgentStore):
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -4,11 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AgentRow(Base):
@@ -37,7 +37,7 @@ class AgentRow(Base):
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class AgentFavoriteRow(Base):
class AgentFavoriteRow(Base):
"""A user's bookmark of a (typically public-square) agent.
Lets a user collect agents from the 广场 (square) into their "我的" tab
@@ -51,4 +51,60 @@ class AgentFavoriteRow(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentPinRow(Base):
"""A per-user pin that keeps an agent near the top of that user's lists."""
__tablename__ = "agent_pins"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentSkillRow(Base):
"""Denormalized cache of a custom agent's selected skills.
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
for the agent runtime (the lead-agent prompt reads it directly). This table
is a read cache so the gallery list endpoint can render skill badges with a
single batched ``IN`` query instead of stat+YAML-parsing every agent's
``config.yaml`` on each request. Written through on create/update and
backfilled from disk by ``_sync_legacy_agents``.
One row per (agent, skill); ``position`` preserves the configured order.
"""
__tablename__ = "agent_skills"
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
skill_name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
position: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
class AgentExtrasRow(Base):
"""Denormalized cache of a custom agent's ``tool_groups`` / ``model``.
Companion to :class:`AgentSkillRow` — same rationale (avoid per-agent
``config.yaml`` reads in the list path). One row per agent. The presence of
a row also marks "this agent has been mirrored": the list path treats a
missing row as "unknown / not yet backfilled" (``skills=None``) so it can
be told apart from an explicitly empty skill list.
``skills_synced_at`` records when the cache was last written from disk so
the backfill can skip agents whose ``config.yaml`` has not changed.
"""
__tablename__ = "agent_extras"
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -6,12 +6,20 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, delete, func, or_, select
from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.tags.model import TagAssignmentRow
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
@@ -233,6 +241,167 @@ class AgentRepository(AgentStore):
await session.commit()
return True
# --- Paginated listing ---
def _scope_filter(self, *, user_id: str, scope: str | None, admin: bool):
"""Build the visibility predicate for a paginated listing."""
if admin:
return None # no visibility filter — every agent
if scope == "square":
return and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
)
if scope == "mine":
# owned OR favorited. A correlated IN-subquery (not a JOIN) keeps a
# single row per agent even when it is both owned and favorited.
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
return or_(AgentRow.user_id == user_id, AgentRow.id.in_(fav_subq))
if scope == "private":
return AgentRow.is_private.is_(True)
return self._listing_filter(user_id)
@staticmethod
def _search_clause(search: str):
"""Match the raw name OR (via the desensitize mirror) its display form."""
needle = search.strip().lower()
clauses = [func.lower(AgentRow.name).like(f"%{needle}%")]
for term in source_terms_for_needle(needle):
clauses.append(AgentRow.name.like(f"%{term}%"))
return or_(*clauses)
@staticmethod
def _tag_clause(tag_ids: list[str]):
subq = select(TagAssignmentRow.target_id).where(
TagAssignmentRow.target_type == "agent",
TagAssignmentRow.tag_id.in_(tag_ids),
)
return AgentRow.id.in_(subq)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
if scope_filter is not None:
filters.append(scope_filter)
if search and search.strip():
filters.append(self._search_clause(search))
if tag_ids:
filters.append(self._tag_clause(tag_ids))
count_stmt = select(func.count()).select_from(AgentRow)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
if sort == "time_asc":
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
elif sort == "updated_desc":
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
elif sort == "updated_asc":
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
elif sort == "name_asc":
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
elif sort == "name_desc":
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
else:
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
page_stmt = (
select(AgentRow)
.order_by(pin_rank.asc(), *order_by)
.limit(page_size)
.offset(max(page - 1, 0) * page_size)
)
if filters:
where = and_(*filters)
count_stmt = count_stmt.where(where)
page_stmt = page_stmt.where(where)
async with self._sf() as session:
total = await session.scalar(count_stmt) or 0
result = await session.execute(page_stmt)
rows = [_row_to_dict(row) for row in result.scalars()]
return rows, total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
for position, name in enumerate(skills or []):
if not isinstance(name, str):
continue
session.add(
AgentSkillRow(
id=uuid4().hex,
agent_id=agent_id,
skill_name=name,
position=position,
)
)
await session.commit()
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
async with self._sf() as session:
row = await session.get(AgentExtrasRow, agent_id)
now = datetime.now(UTC)
if row is None:
session.add(
AgentExtrasRow(
agent_id=agent_id,
tool_groups=tool_groups,
model=model,
skills_synced_at=now,
)
)
else:
row.tool_groups = tool_groups
row.model = model
row.skills_synced_at = now
await session.commit()
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
if not agent_ids:
return {}
out: dict[str, dict[str, Any]] = {}
async with self._sf() as session:
extras = await session.execute(
select(AgentExtrasRow).where(AgentExtrasRow.agent_id.in_(agent_ids))
)
for row in extras.scalars():
out[row.agent_id] = {
"skills": [], # default; filled below if skill rows exist
"tool_groups": row.tool_groups,
"model": row.model,
}
skill_rows = await session.execute(
select(AgentSkillRow.agent_id, AgentSkillRow.skill_name)
.where(AgentSkillRow.agent_id.in_(agent_ids))
.order_by(AgentSkillRow.agent_id.asc(), AgentSkillRow.position.asc())
)
for agent_id, skill_name in skill_rows.all():
entry = out.setdefault(
agent_id, {"skills": [], "tool_groups": None, "model": None}
)
entry["skills"].append(skill_name)
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
await session.execute(delete(AgentExtrasRow).where(AgentExtrasRow.agent_id == agent_id))
await session.commit()
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
@@ -272,7 +441,49 @@ class AgentRepository(AgentStore):
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentPinRow.id)
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentPinRow).where(
AgentPinRow.user_id == user_id,
AgentPinRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_pin_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_pins_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
await session.commit()
@@ -11,16 +11,16 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
def _json_serializer(obj: object) -> str:
@@ -133,7 +133,7 @@ async def _run_alembic_upgrade(url: str) -> None:
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
async def _auto_create_postgres_db(url: str) -> None:
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
The target database name is extracted from *url*. The connection is
@@ -156,55 +156,57 @@ async def _auto_create_postgres_db(url: str) -> None:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
backend: str,
*,
url: str = "",
echo: bool = False,
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
@@ -213,7 +215,9 @@ async def init_engine(
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
echo: Echo SQL to log.
pool_size: Postgres connection pool size.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory
@@ -271,6 +275,8 @@ async def init_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
@@ -279,6 +285,8 @@ async def init_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
@@ -302,37 +310,39 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
@@ -351,6 +361,8 @@ async def init_engine_from_config(config) -> None:
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
@@ -34,6 +34,9 @@ class LlmMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
# Multiple user ids — set when a fuzzy username search resolves to several
# accounts. Takes precedence over ``user_id`` when non-empty.
user_ids: list[str] | None = None
model_name: str | None = None
status: str | None = None
since: datetime | None = None
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, Text
from sqlalchemy import Float, Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -15,6 +15,10 @@ class LlmCallMetricRow(Base):
"""One row per completed (or failed) LLM provider invocation."""
__tablename__ = "llm_call_metrics"
__table_args__ = (
Index("ix_llm_call_metrics_created_user", "created_at", "user_id"),
Index("ix_llm_call_metrics_run_created", "run_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
@@ -58,7 +58,9 @@ class SqlLlmMetricsStore(LlmMetricsStore):
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: LlmMetricsQuery):
if query.user_id:
if query.user_ids:
stmt = stmt.where(LlmCallMetricRow.user_id.in_(query.user_ids))
elif query.user_id:
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
if query.model_name:
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
@@ -92,6 +94,7 @@ class SqlLlmMetricsStore(LlmMetricsStore):
while True:
page_query = LlmMetricsQuery(
user_id=query.user_id,
user_ids=query.user_ids,
model_name=query.model_name,
status=query.status,
since=query.since,
@@ -0,0 +1,45 @@
"""Add ``agent_id`` to ``user_custom_prompts`` for per-agent prompt scoping.
NULL = global (shown for every agent); otherwise the prompt only surfaces when
that agent is selected.
Revision ID: 20260603_02
Revises: 20260603_01
Create Date: 2026-06-03
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260603_02"
down_revision = "20260603_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("user_custom_prompts"):
return
if not _has_column("user_custom_prompts", "agent_id"):
op.add_column("user_custom_prompts", sa.Column("agent_id", sa.String(64), nullable=True))
op.create_index("ix_user_custom_prompts_agent_id", "user_custom_prompts", ["agent_id"])
def downgrade() -> None:
if _has_table("user_custom_prompts") and _has_column("user_custom_prompts", "agent_id"):
op.drop_index("ix_user_custom_prompts_agent_id", table_name="user_custom_prompts")
op.drop_column("user_custom_prompts", "agent_id")
@@ -0,0 +1,42 @@
"""Add ``user_preferences`` for per-user UI toggles (account-bound).
Holds a single JSON blob per user — e.g. ``hide_admin_prompt_prefix`` so a user
can choose to use only their own prompts and hide the admin-configured prefix.
Revision ID: 20260603_03
Revises: 20260603_02
Create Date: 2026-06-03
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260603_03"
down_revision = "20260603_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("user_preferences"):
op.create_table(
"user_preferences",
sa.Column("user_id", sa.String(64), primary_key=True),
sa.Column("preferences", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if _has_table("user_preferences"):
op.drop_table("user_preferences")
@@ -0,0 +1,56 @@
"""Per-user pinned agents.
Revision ID: 20260604_01
Revises: 20260603_03
Create Date: 2026-06-04
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260604_01"
down_revision = "20260603_03"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if not _has_table("agent_pins"):
op.create_table(
"agent_pins",
sa.Column("id", sa.String(64), nullable=False),
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column("agent_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),
)
if not _has_index("agent_pins", "ix_agent_pins_user_id"):
op.create_index("ix_agent_pins_user_id", "agent_pins", ["user_id"])
if not _has_index("agent_pins", "ix_agent_pins_agent_id"):
op.create_index("ix_agent_pins_agent_id", "agent_pins", ["agent_id"])
def downgrade() -> None:
if _has_index("agent_pins", "ix_agent_pins_agent_id"):
op.drop_index("ix_agent_pins_agent_id", table_name="agent_pins")
if _has_index("agent_pins", "ix_agent_pins_user_id"):
op.drop_index("ix_agent_pins_user_id", table_name="agent_pins")
if _has_table("agent_pins"):
op.drop_table("agent_pins")
@@ -7,7 +7,7 @@ MySQL (can exceed the 64 KB TEXT cap) and plain TEXT elsewhere — matching
``PortableLongText`` on ``RoundtableJobRow``. ``create_all`` adds the table for
brand-new DBs; this revision creates it on DBs that predate it.
Revision ID: 20260604_01
Revision ID: 20260604_02
Revises: 20260603_01
Create Date: 2026-06-04
"""
@@ -20,7 +20,7 @@ from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260604_01"
revision = "20260604_02"
down_revision = "20260603_01"
branch_labels = None
depends_on = None
@@ -0,0 +1,46 @@
"""Add dialogues column to roundtable_jobs.
存研讨对话记录(leader 说明 + 各席位交付),使后台作业跑完 / 进行中用户都能看到
真实研讨内容(而非只有流程图进度)。LONGTEXT(对话可超 64 KB),匹配
``PortableLongText`` on ``RoundtableJobRow.dialogues``。
Revision ID: 20260605_01
Revises: 20260604_01
Create Date: 2026-06-05
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260605_01"
down_revision = "20260604_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_jobs") and not _has_column("roundtable_jobs", "dialogues"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_jobs", sa.Column("dialogues", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_jobs", "dialogues"):
op.drop_column("roundtable_jobs", "dialogues")
@@ -0,0 +1,46 @@
"""Add step3 column to roundtable_jobs.
把作业的 Step3 报告({html, generatedAt, model, summary})直接挂在 job 上,使前端
「多运行」模型(draft.step2.runs[])能从单个 job 读到完整结果(对话 + 报告)组装一个 run,
而不必依赖草稿顶层的单一 step3。LONGTEXT(HTML 可超 64 KB)。
Revision ID: 20260605_02
Revises: 20260605_01
Create Date: 2026-06-05
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260605_02"
down_revision = "20260605_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_jobs") and not _has_column("roundtable_jobs", "step3"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_jobs", sa.Column("step3", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_jobs", "step3"):
op.drop_column("roundtable_jobs", "step3")
@@ -0,0 +1,57 @@
"""Create roundtable_draft_shares table.
Per-draft share links for the roundtable-planning page, mirroring
``thread_shares`` but pointing at a ``roundtable_drafts`` row. ``import`` codes
copy the draft into the recipient's account; ``view`` codes back a public
read-only result-report page. ``create_all`` makes the table for brand-new DBs;
this revision patches DBs that already exist.
Revision ID: 20260606_01
Revises: 20260605_02
Create Date: 2026-06-06
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260606_01"
down_revision = "20260605_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("roundtable_draft_shares"):
return
op.create_table(
"roundtable_draft_shares",
sa.Column("share_code", sa.String(length=32), primary_key=True),
sa.Column("owner_user_id", sa.String(length=64), nullable=False),
sa.Column("draft_id", sa.String(length=64), nullable=False),
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("mode", sa.String(length=16), nullable=False, server_default="import"),
sa.Column("created_at", sa.DateTime(), nullable=False),
)
op.create_index(
"ix_roundtable_draft_shares_owner_user_id",
"roundtable_draft_shares",
["owner_user_id"],
)
op.create_index(
"ix_roundtable_draft_shares_draft_id",
"roundtable_draft_shares",
["draft_id"],
)
def downgrade() -> None:
if _has_table("roundtable_draft_shares"):
op.drop_table("roundtable_draft_shares")
@@ -0,0 +1,47 @@
"""Add ``agent_id`` to ``user_custom_prompts`` when the parallel branch was skipped.
Some deployments reached ``20260606_01`` via the roundtable-jobs migration branch
(``20260603_01`` → ``20260604_01`` jobs → …) without running the sibling branch
that adds ``agent_id`` (``20260603_02``). This idempotent fixup applies the
missing column for those DBs.
Revision ID: 20260607_01
Revises: 20260606_01
Create Date: 2026-06-07
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260607_01"
down_revision = "20260606_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("user_custom_prompts"):
return
if not _has_column("user_custom_prompts", "agent_id"):
op.add_column("user_custom_prompts", sa.Column("agent_id", sa.String(64), nullable=True))
op.create_index("ix_user_custom_prompts_agent_id", "user_custom_prompts", ["agent_id"])
def downgrade() -> None:
if _has_table("user_custom_prompts") and _has_column("user_custom_prompts", "agent_id"):
op.drop_index("ix_user_custom_prompts_agent_id", table_name="user_custom_prompts")
op.drop_column("user_custom_prompts", "agent_id")
@@ -0,0 +1,22 @@
"""Merge parallel migration branches (agent_pins + roundtable jobs fixup).
Revision ID: 20260607_02
Revises: 20260604_01, 20260607_01
Create Date: 2026-06-07
"""
from __future__ import annotations
revision = "20260607_02"
down_revision = ("20260604_01", "20260607_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,49 @@
"""Add stages column to roundtable_chains.
Layered orchestration (分层编排) for business chains: an optional list-of-lists of
agent_ids grouping the chain's seats into parallel stages (stages run serially).
NULL = linear chain. The column is LONGTEXT on MySQL (mirrors ``seats`` headroom)
and plain TEXT elsewhere — matching ``PortableLongText`` on
``RoundtableChainRow.stages``. ``create_all`` adds the column for brand-new DBs;
this revision patches DBs that already have the table.
Revision ID: 20260607_03
Revises: 20260607_02
Create Date: 2026-06-07
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260607_03"
down_revision = "20260607_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_chains") and not _has_column("roundtable_chains", "stages"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_chains", sa.Column("stages", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_chains", "stages"):
op.drop_column("roundtable_chains", "stages")
@@ -0,0 +1,85 @@
"""Create admin leaderboard daily snapshots and analytics indexes.
Revision ID: 20260608_01
Revises: 20260607_02
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from deerflow.persistence.types import PortableJSON
revision = "20260608_01"
down_revision = "20260607_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def _create_index(table_name: str, index_name: str, columns: list[str]) -> None:
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, columns)
def _drop_index(table_name: str, index_name: str) -> None:
if _has_table(table_name) and _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
def upgrade() -> None:
if not _has_table("admin_leaderboard_daily_stats"):
op.create_table(
"admin_leaderboard_daily_stats",
sa.Column("stat_date", sa.String(10), nullable=False),
sa.Column("settings_hash", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False, server_default="ready"),
sa.Column("payload_json", PortableJSON(), nullable=True),
sa.Column("message", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("stat_date", "settings_hash", name="pk_admin_leaderboard_daily_stats"),
)
op.create_index("ix_admin_leaderboard_daily_stats_status", "admin_leaderboard_daily_stats", ["status"])
op.create_index("ix_admin_leaderboard_daily_stats_updated_at", "admin_leaderboard_daily_stats", ["updated_at"])
_create_index("runs", "ix_runs_created_user_status", ["created_at", "user_id", "status"])
_create_index("runs", "ix_runs_user_created", ["user_id", "created_at"])
_create_index("runs", "ix_runs_thread_created", ["thread_id", "created_at"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_created_user", ["created_at", "user_id"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_run_created", ["run_id", "created_at"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_created_user", ["created_at", "user_id"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_run_created", ["run_id", "created_at"])
_create_index("threads_meta", "ix_threads_meta_created_user", ["created_at", "user_id"])
_create_index("threads_meta", "ix_threads_meta_user_created", ["user_id", "created_at"])
def downgrade() -> None:
_drop_index("threads_meta", "ix_threads_meta_user_created")
_drop_index("threads_meta", "ix_threads_meta_created_user")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_run_created")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_created_user")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_run_created")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_created_user")
_drop_index("runs", "ix_runs_thread_created")
_drop_index("runs", "ix_runs_user_created")
_drop_index("runs", "ix_runs_created_user_status")
if _has_table("admin_leaderboard_daily_stats"):
op.drop_table("admin_leaderboard_daily_stats")
@@ -0,0 +1,27 @@
"""Merge chain_stages and admin_leaderboard_daily_stats migration heads.
Two independent migrations diverged from ``20260607_02``: ``20260607_03``
(roundtable_chains.stages) and ``20260608_01`` (admin_leaderboard_daily_stats +
analytics indexes). This no-op merge revision joins both lineages so
``upgrade head`` succeeds.
Revision ID: 20260608_02
Revises: 20260607_03, 20260608_01
Create Date: 2026-06-08
"""
from __future__ import annotations
revision = "20260608_02"
down_revision = ("20260607_03", "20260608_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,58 @@
"""Add is_public column to roundtable_chains.
「发布」business chains: ``is_public=True`` makes a chain visible to **all
accounts** under the "公共" filter (read + select-into-roundtable only; edit /
delete stay owner-only). Defaults to ``0`` (private). ``create_all`` adds the
column for brand-new DBs; this revision patches DBs that already have the table.
Chained after the merge head ``20260608_02`` (chain_stages + leaderboard) so the
revision graph stays single-headed — otherwise ``alembic upgrade head`` aborts
with "multiple heads" and the auto-migrate on startup silently falls back to the
create_all schema (which lacks this column → 500 on list).
Revision ID: 20260608_03
Revises: 20260608_02
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260608_03"
down_revision = "20260608_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_chains") and not _has_column("roundtable_chains", "is_public"):
op.add_column(
"roundtable_chains",
sa.Column("is_public", sa.Boolean(), nullable=False, server_default="0"),
)
op.create_index(
"ix_roundtable_chains_is_public", "roundtable_chains", ["is_public"]
)
def downgrade() -> None:
if _has_column("roundtable_chains", "is_public"):
with op.batch_alter_table("roundtable_chains") as batch:
batch.drop_index("ix_roundtable_chains_is_public")
op.drop_column("roundtable_chains", "is_public")
@@ -0,0 +1,56 @@
"""Add stage_goals column to roundtable_chains.
Per-stage goals (每阶段目标) for business chains: an optional list aligned by index
to ``stages`` where ``stage_goals[i]`` is the「本阶段目标」for ``stages[i]`` — what
that serial stage should produce / how it depends on the previous stage's output.
Injected to the coordinator dispatch prompt (and seat tasks as fallback) in dag-mode
only. NULL = no goals. The column is LONGTEXT on MySQL (mirrors ``stages`` headroom)
and plain TEXT elsewhere — matching ``PortableLongText`` on
``RoundtableChainRow.stage_goals``. ``create_all`` adds the column for brand-new DBs;
this revision patches DBs that already have the table.
Chained after ``20260608_03`` (chain is_public) so the revision graph stays
single-headed — otherwise ``alembic upgrade head`` aborts with "multiple heads" and
the auto-migrate on startup silently falls back to the create_all schema (which lacks
this column → 500 on list).
Revision ID: 20260608_04
Revises: 20260608_03
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260608_04"
down_revision = "20260608_03"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_chains") and not _has_column("roundtable_chains", "stage_goals"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_chains", sa.Column("stage_goals", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_chains", "stage_goals"):
op.drop_column("roundtable_chains", "stage_goals")
@@ -0,0 +1,61 @@
"""Create parallel_agent_tasks table.
Saved runs of the standalone parallel multi-agent panel (Vue embeddable
component): task name + brief + agents snapshot + the resulting message list,
bound per user. ``brief`` / ``agents`` / ``messages`` are LONGTEXT on MySQL
(mirror ``roundtable_drafts`` headroom) and plain TEXT elsewhere — matching
``PortableLongText``. ``create_all`` builds the table for brand-new DBs; this
revision patches DBs that already exist.
Chained after ``20260608_04`` (chain stage_goals) so the revision graph stays
single-headed — otherwise ``alembic upgrade head`` aborts with "multiple heads"
and the startup auto-migrate silently falls back to the create_all schema.
Revision ID: 20260608_05
Revises: 20260608_04
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260608_05"
down_revision = "20260608_04"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("parallel_agent_tasks"):
return
long_text = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.create_table(
"parallel_agent_tasks",
sa.Column("id", sa.String(length=64), primary_key=True, nullable=False),
sa.Column("user_id", sa.String(length=64), nullable=False),
sa.Column("title", sa.String(length=512), nullable=False, server_default=""),
sa.Column("brief", long_text, nullable=True),
sa.Column("agents", long_text, nullable=True),
sa.Column("messages", long_text, nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_index(
"ix_parallel_agent_tasks_user_id", "parallel_agent_tasks", ["user_id"]
)
def downgrade() -> None:
if _has_table("parallel_agent_tasks"):
op.drop_index("ix_parallel_agent_tasks_user_id", table_name="parallel_agent_tasks")
op.drop_table("parallel_agent_tasks")
@@ -0,0 +1,56 @@
"""Add coordinator_prompt column to roundtable_chains.
Per-chain「总控编排提示」: an optional free-text instruction the user writes for the
**coordinator (总控) agent** of a business chain — telling it how to dispatch / what to
do (overall orchestration strategy / style / constraints). Injected into the coordinator
per-stage dispatch prompt (``buildDagStageLeaderTask``) in **dag-mode only**. NULL = no
prompt. The column is LONGTEXT on MySQL (mirrors ``stage_goals`` headroom) and plain TEXT
elsewhere — matching ``PortableLongText`` on ``RoundtableChainRow.coordinator_prompt``.
``create_all`` adds the column for brand-new DBs; this revision patches DBs that already
have the table.
Chained after ``20260608_05`` (parallel_agent_tasks) so the revision graph stays
single-headed — otherwise ``alembic upgrade head`` aborts with "multiple heads" and the
startup auto-migrate silently falls back to the create_all schema (which lacks this
column → 500 on list).
Revision ID: 20260608_06
Revises: 20260608_05
Create Date: 2026-06-09
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260608_06"
down_revision = "20260608_05"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_table("roundtable_chains") and not _has_column("roundtable_chains", "coordinator_prompt"):
col_type = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
op.add_column("roundtable_chains", sa.Column("coordinator_prompt", col_type, nullable=True))
def downgrade() -> None:
if _has_column("roundtable_chains", "coordinator_prompt"):
op.drop_column("roundtable_chains", "coordinator_prompt")
@@ -15,14 +15,22 @@ its storage implementation lives in ``deerflow.runtime.events.store.db`` and
there is no matching entity directory.
"""
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.roundtable_draft_shares.model import RoundtableDraftShareRow
from deerflow.persistence.roundtable_drafts.model import (
RoundtableDraftRow,
RoundtableRecommendHistoryRow,
@@ -43,17 +51,23 @@ from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
__all__ = [
"FeedbackRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"LlmCallMetricRow",
"NotificationRow",
"ParallelAgentTaskRow",
"RecommendedQuestionRow",
"RoundtableChainRow",
"RoundtableDraftRow",
"RoundtableDraftShareRow",
"RoundtableJobRow",
"RoundtableRecommendHistoryRow",
"RunEventRow",
@@ -73,5 +87,6 @@ __all__ = [
"ToolCallMetricRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
"UserPreferencesRow",
"UserRow",
]
@@ -0,0 +1,18 @@
"""Parallel multi-agent task persistence."""
from __future__ import annotations
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
from deerflow.persistence.parallel_agent_tasks.sql import ParallelAgentTaskRepository
__all__ = [
"ParallelAgentTaskRepository",
"ParallelAgentTaskRow",
"make_parallel_agent_task_store",
]
def make_parallel_agent_task_store(session_factory):
if session_factory is None:
return None
return ParallelAgentTaskRepository(session_factory)
@@ -0,0 +1,46 @@
"""ORM model for parallel multi-agent task records.
A "parallel-agent task" is one saved run of the standalone *parallel multi-agent*
panel (the Vue3 embeddable component): a task name + brief dispatched to several
agents in parallel, plus the resulting chat/message list. Persisted per user so it
follows the account across devices/sessions — same model as ``roundtable_drafts``.
``agents`` and ``messages`` are stored as JSON serialized into ``PortableLongText``
(MySQL ``LONGTEXT``) rather than ``PortableJSON`` (MySQL ``TEXT``, 64 KB cap),
because the parallel transcript can carry several agents' full deliveries that
easily exceed 64 KB — mirroring ``roundtable_drafts``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class ParallelAgentTaskRow(Base):
__tablename__ = "parallel_agent_tasks"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Task name (用户输入的任务名称) shown on the history list.
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# Task brief / 简介 — the free-text task description.
brief: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# JSON: snapshot of the agents dispatched, [{"agent_id", "name", "avatarType"?}].
agents: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# JSON: the full message/dialogue list (mirrors the frontend Dialogue[] shape).
messages: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,131 @@
"""SQLAlchemy-backed store for parallel multi-agent task records.
``agents`` / ``messages`` JSON columns are stored as serialized strings in
``PortableLongText`` and round-tripped here with ``json.dumps`` / ``json.loads``
— same approach as ``roundtable_drafts`` / ``roundtable_chains``.
All write/read paths are scoped by ``user_id`` for ownership isolation; a row
that belongs to another user is treated as not found.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
_UNSET = object()
def _loads(value: Any) -> Any:
if not isinstance(value, str):
return value
if not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class ParallelAgentTaskRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
data: dict[str, Any] = {
"id": row.id,
"title": row.title,
"brief": row.brief,
"agents": _loads(row.agents) or [],
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
if with_messages:
data["messages"] = _loads(row.messages) or []
return data
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
stmt = (
select(ParallelAgentTaskRow)
.where(ParallelAgentTaskRow.user_id == user_id)
.order_by(ParallelAgentTaskRow.updated_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ParallelAgentTaskRow, task_id)
if row is None or row.user_id != user_id:
return None
return self._to_dict(row)
async def create_task(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = ParallelAgentTaskRow(
id=data["id"],
user_id=user_id,
title=(data.get("title") or "").strip(),
brief=(data.get("brief") or None),
agents=_dumps(data.get("agents") or []),
messages=_dumps(data.get("messages") or []),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def update_task(
self,
task_id: str,
user_id: str,
*,
title=_UNSET,
brief=_UNSET,
agents=_UNSET,
messages=_UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ParallelAgentTaskRow, task_id)
if row is None or row.user_id != user_id:
return None
if title is not _UNSET and title is not None:
row.title = str(title).strip()
if brief is not _UNSET:
row.brief = brief or None
if agents is not _UNSET:
row.agents = _dumps(agents)
if messages is not _UNSET:
row.messages = _dumps(messages)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def delete_task(self, task_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ParallelAgentTaskRow, task_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
@@ -14,7 +14,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -33,6 +33,27 @@ class RoundtableChainRow(Base):
# JSON: ordered [{"agent_id": str, "name": str, "description": str}, ...].
# seats[0] is the first seat to speak. Stored as LONGTEXT-backed JSON string.
seats: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Optional JSON: layered orchestration grouping over ``seats`` — a list of
# stages, each a list of agent_ids run in parallel; stages run serially in
# order. e.g. [["a","b"],["c"]] = (a‖b) → c. NULL/absent = linear chain
# (each seat its own stage). Drives Step 2 dag-mode orchestration; see
# frontend ``lib/dag-types.ts``. Stored as LONGTEXT-backed JSON string.
stages: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Optional JSON: per-stage goals aligned by index to ``stages`` — a list where
# ``stage_goals[i]`` is the「本阶段目标」for ``stages[i]`` (what that serial stage
# should produce / how it depends on the previous stage's output). Injected to the
# coordinator dispatch prompt (and seat tasks as fallback) in dag-mode only. NULL =
# no goals (and always NULL when ``stages`` is NULL/linear). Stored as JSON string.
stage_goals: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 可选「总控编排提示」:用户为本链条写给**总控(协调)智能体**的一段指引,告诉总控该
# 怎么派活、做什么(整体编排策略 / 风格 / 约束)。**仅分层 DAG 模式生效**:注入到总控
# 每阶段派活提示(``buildDagStageLeaderTask``)。NULL = 不注入。纯文本,存为 LONGTEXT。
coordinator_prompt: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 发布开关:True = 公共(所有账号可在「公共」筛选里查看 / 选用);False = 仅本人可见。
# 编辑 / 删除始终只限 owner;发布只是放开「查看 + 选入研讨」。索引以支持「公共」列表筛选。
is_public: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0", index=True
)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -44,33 +44,58 @@ class RoundtableChainRepository:
self._sf = session_factory
@staticmethod
def _to_dict(row: RoundtableChainRow) -> dict[str, Any]:
def _to_dict(row: RoundtableChainRow, current_user_id: str | None = None) -> dict[str, Any]:
return {
"id": row.id,
"title": row.title,
"description": row.description,
"seats": _loads(row.seats) or [],
# None when absent → frontend reads it as "linear chain" (no layering).
"stages": _loads(row.stages),
# Per-stage goals aligned to ``stages``; None when absent.
"stage_goals": _loads(getattr(row, "stage_goals", None)),
# 总控编排提示(写给总控的派活指引,dag-only);None/空 = 不注入。纯文本,不 json。
"coordinator_prompt": getattr(row, "coordinator_prompt", None),
# 发布状态 + 是否本人所有(前端据此显示「公共」徽标、决定能否编辑 / 发布)。
"is_public": bool(getattr(row, "is_public", False)),
"is_mine": (current_user_id is None) or (row.user_id == current_user_id),
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
async def list_chains(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
async def list_chains(
self, user_id: str, *, scope: str = "mine", limit: int = 200
) -> list[dict[str, Any]]:
"""列出链条。
``scope``:
- ``"mine"``(默认):仅当前用户自己的链条;
- ``"public"``:所有**已发布**(``is_public=True``)链条,含本人发布的。
其余值按 ``"mine"`` 处理。``is_mine`` 标记每条是否本人所有,供前端判断可否编辑。
"""
if scope == "public":
cond = RoundtableChainRow.is_public.is_(True)
else:
cond = RoundtableChainRow.user_id == user_id
stmt = (
select(RoundtableChainRow)
.where(RoundtableChainRow.user_id == user_id)
.where(cond)
.order_by(RoundtableChainRow.updated_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row) for row in result.scalars()]
return [self._to_dict(row, user_id) for row in result.scalars()]
async def get_chain(self, chain_id: str, user_id: str) -> dict[str, Any] | None:
"""取一条链条:owner 可见,**已发布的公共链条任何人可见**(供选入研讨)。"""
async with self._sf() as session:
row = await session.get(RoundtableChainRow, chain_id)
if row is None or row.user_id != user_id:
if row is None:
return None
return self._to_dict(row)
if row.user_id != user_id and not bool(getattr(row, "is_public", False)):
return None
return self._to_dict(row, user_id)
async def create_chain(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
@@ -80,6 +105,10 @@ class RoundtableChainRepository:
title=(data.get("title") or "").strip(),
description=(data.get("description") or None),
seats=_dumps(data.get("seats") or []),
stages=_dumps(data.get("stages")),
stage_goals=_dumps(data.get("stage_goals")),
coordinator_prompt=(data.get("coordinator_prompt") or None),
is_public=bool(data.get("is_public") or False),
created_at=now,
updated_at=now,
)
@@ -87,7 +116,7 @@ class RoundtableChainRepository:
session.add(row)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
return self._to_dict(row, user_id)
async def update_chain(
self,
@@ -97,6 +126,10 @@ class RoundtableChainRepository:
title=_UNSET,
description=_UNSET,
seats=_UNSET,
stages=_UNSET,
stage_goals=_UNSET,
coordinator_prompt=_UNSET,
is_public=_UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableChainRow, chain_id)
@@ -108,10 +141,18 @@ class RoundtableChainRepository:
row.description = description or None
if seats is not _UNSET:
row.seats = _dumps(seats)
if stages is not _UNSET:
row.stages = _dumps(stages)
if stage_goals is not _UNSET:
row.stage_goals = _dumps(stage_goals)
if coordinator_prompt is not _UNSET:
row.coordinator_prompt = coordinator_prompt or None
if is_public is not _UNSET and is_public is not None:
row.is_public = bool(is_public)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
return self._to_dict(row, user_id)
async def delete_chain(self, chain_id: str, user_id: str) -> bool:
async with self._sf() as session:
@@ -0,0 +1,22 @@
"""Persistence layer for roundtable-draft share links."""
from deerflow.persistence.roundtable_draft_shares.base import (
RoundtableDraftShareStore,
generate_share_code,
)
from deerflow.persistence.roundtable_draft_shares.memory import MemoryRoundtableDraftShareStore
from deerflow.persistence.roundtable_draft_shares.sql import RoundtableDraftShareRepository
__all__ = [
"RoundtableDraftShareStore",
"MemoryRoundtableDraftShareStore",
"RoundtableDraftShareRepository",
"generate_share_code",
"make_roundtable_draft_share_store",
]
def make_roundtable_draft_share_store(session_factory):
if session_factory is None:
return MemoryRoundtableDraftShareStore()
return RoundtableDraftShareRepository(session_factory)
@@ -0,0 +1,56 @@
"""Abstract persistence API for roundtable-draft share links.
Mirrors ``thread_shares`` but shares one *roundtable-planning draft* (a full
会商 session: Step 1 intent + Step 2 roundtable + optional Step 3 report)
instead of a conversation thread. Redeeming an ``import`` code copies the draft
into the recipient's account; a ``view`` code is a public read-only page that
renders the draft's Step 3 result report.
The share-code alphabet / generator is reused from ``thread_shares.base`` so
codes are interchangeable in shape (unambiguous, human-readable).
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
# Reuse the same unambiguous-alphabet code generator as conversation shares.
from deerflow.persistence.thread_shares.base import generate_share_code
__all__ = ["RoundtableDraftShareStore", "generate_share_code"]
class RoundtableDraftShareStore(ABC):
@abstractmethod
async def create(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any]:
"""Create a new share link for one draft and return its record.
``mode`` is ``"import"`` (recipient copies the draft) or ``"view"``
(a public read-only result-report page).
"""
raise NotImplementedError
@abstractmethod
async def get(self, share_code: str) -> dict[str, Any] | None:
"""Look up a share by code regardless of owner (None if missing)."""
raise NotImplementedError
@abstractmethod
async def get_active_for_draft(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any] | None:
"""Return the owner's non-revoked share of ``mode`` for ``draft_id``, if any.
Used so re-sharing the same draft in the same mode reuses its existing
code instead of minting a new one each time.
"""
raise NotImplementedError
@abstractmethod
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
"""List all share links created by ``owner_user_id`` (newest first)."""
raise NotImplementedError
@abstractmethod
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
"""Revoke a share link. Returns False if missing or not owned."""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""In-memory ``RoundtableDraftShareStore`` for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.roundtable_draft_shares.base import RoundtableDraftShareStore, generate_share_code
class MemoryRoundtableDraftShareStore(RoundtableDraftShareStore):
def __init__(self) -> None:
self._shares: dict[str, dict[str, Any]] = {}
async def create(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any]:
code = generate_share_code()
while code in self._shares:
code = generate_share_code()
record = {
"share_code": code,
"owner_user_id": owner_user_id,
"draft_id": draft_id,
"revoked": False,
"mode": mode,
"created_at": datetime.now(UTC).isoformat(),
}
self._shares[code] = record
return copy.deepcopy(record)
async def get(self, share_code: str) -> dict[str, Any] | None:
record = self._shares.get(share_code)
return copy.deepcopy(record) if record is not None else None
async def get_active_for_draft(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any] | None:
matches = [
r
for r in self._shares.values()
if r["owner_user_id"] == owner_user_id
and r["draft_id"] == draft_id
and r.get("mode", "import") == mode
and not r["revoked"]
]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
rows.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in rows]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
record = self._shares.get(share_code)
if record is None or record["owner_user_id"] != owner_user_id:
return False
record["revoked"] = True
return True
@@ -0,0 +1,38 @@
"""ORM model for roundtable-draft share links.
A share link is a short, revocable code created by a user for *one* of their
roundtable-planning drafts. Redeeming it copies that draft into the redeemer's
account (``import``) or exposes its Step 3 result report on a public read-only
page (``view``). The table only stores the link — the draft snapshot is read
lazily at import / view time, so a revoked link instantly stops working.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class RoundtableDraftShareRow(Base):
__tablename__ = "roundtable_draft_shares"
# Short, URL-safe, unguessable code embedded in the share link.
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# The user who created the share.
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# The single roundtable draft this link shares.
draft_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Soft-disable: a revoked link can no longer be previewed, imported, or viewed.
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Share mode:
# "import" — recipient (a logged-in user) copies the draft into their own
# account and can keep working on the copy.
# "view" — a public, read-only page anyone can open without logging in;
# it renders the draft's Step 3 result report (HTML).
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,84 @@
"""SQLAlchemy-backed roundtable-draft share-link repository."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.roundtable_draft_shares.base import RoundtableDraftShareStore, generate_share_code
from deerflow.persistence.roundtable_draft_shares.model import RoundtableDraftShareRow
def _row_to_dict(row: RoundtableDraftShareRow) -> dict[str, Any]:
data = row.to_dict()
created = data.get("created_at")
if isinstance(created, datetime):
data["created_at"] = created.isoformat()
return data
class RoundtableDraftShareRepository(RoundtableDraftShareStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any]:
# Retry on the (astronomically unlikely) code collision.
async with self._sf() as session:
for _ in range(8):
code = generate_share_code()
if await session.get(RoundtableDraftShareRow, code) is not None:
continue
row = RoundtableDraftShareRow(
share_code=code,
owner_user_id=owner_user_id,
draft_id=draft_id,
revoked=False,
mode=mode,
)
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
raise RuntimeError("Failed to allocate a unique share code")
async def get(self, share_code: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(RoundtableDraftShareRow, share_code)
return _row_to_dict(row) if row is not None else None
async def get_active_for_draft(self, owner_user_id: str, draft_id: str, *, mode: str = "import") -> dict[str, Any] | None:
stmt = (
select(RoundtableDraftShareRow)
.where(
RoundtableDraftShareRow.owner_user_id == owner_user_id,
RoundtableDraftShareRow.draft_id == draft_id,
RoundtableDraftShareRow.mode == mode,
RoundtableDraftShareRow.revoked.is_(False),
)
.order_by(RoundtableDraftShareRow.created_at.desc())
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalars().first()
return _row_to_dict(row) if row is not None else None
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
stmt = (
select(RoundtableDraftShareRow)
.where(RoundtableDraftShareRow.owner_user_id == owner_user_id)
.order_by(RoundtableDraftShareRow.created_at.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(RoundtableDraftShareRow, share_code)
if row is None or row.owner_user_id != owner_user_id:
return False
row.revoked = True
await session.commit()
return True
@@ -55,6 +55,12 @@ class RoundtableJobRow(Base):
orchestration_mode: Mapped[str] = mapped_column(String(16), nullable=False, default="recommend")
# chain 模式来源链条快照:JSON object { id, title }(recommend 模式为 null).
chain: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 研讨对话记录:JSON array of { id, role('leader'|'seat'), agentId, name, content, cycle }。
# 让用户跑完 / 进行中都能看到真实研讨内容(leader 说明 + 各席位交付),而非只有流程图。
dialogues: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Step3 报告快照:JSON object { html, generatedAt, model, summary }。done 时写入,
# 使前端从单个 job 即可拿到完整结果(对话 + 报告)。
step3: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Terminal error detail when status=error.
error: Mapped[str | None] = mapped_column(Text, nullable=True)
# 时间列必须用 BeijingDateTime(项目规范,与 ai_writing_sessions / roundtable_drafts 对齐)。
@@ -55,7 +55,7 @@ class RoundtableJobRepository:
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
for k in ("dispatch_chain", "thread_ids", "chain"):
for k in ("dispatch_chain", "thread_ids", "chain", "dialogues", "step3"):
d[k] = _loads(d.get(k))
return d
@@ -118,6 +118,8 @@ class RoundtableJobRepository:
pending_clarification=_UNSET,
thread_ids=_UNSET,
coordinator_name=_UNSET,
dialogues=_UNSET,
step3=_UNSET,
error=_UNSET,
) -> dict[str, Any] | None:
resolved_user_id = resolve_user_id(user_id, method_name="RoundtableJobRepository.update_progress")
@@ -145,6 +147,10 @@ class RoundtableJobRepository:
row.thread_ids = _dumps(thread_ids)
if coordinator_name is not _UNSET:
row.coordinator_name = coordinator_name
if dialogues is not _UNSET:
row.dialogues = _dumps(dialogues)
if step3 is not _UNSET:
row.step3 = _dumps(step3)
if error is not _UNSET:
row.error = error
row.updated_at = datetime.now(UTC)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -47,4 +47,9 @@ class RunRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
__table_args__ = (
Index("ix_runs_thread_status", "thread_id", "status"),
Index("ix_runs_created_user_status", "created_at", "user_id", "status"),
Index("ix_runs_user_created", "user_id", "created_at"),
Index("ix_runs_thread_created", "thread_id", "created_at"),
)
@@ -205,6 +205,21 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
Each run is one user question, so this is the source for the admin
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
"""
stmt = (
select(RunRow.user_id, func.count().label("cnt"))
.where(RunRow.user_id.is_not(None))
.group_by(RunRow.user_id)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return {r.user_id: r.cnt for r in rows}
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
"""Aggregate token usage via a single SQL GROUP BY query."""
_completed = RunRow.status.in_(("success", "error"))
@@ -4,15 +4,19 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class ThreadMetaRow(Base):
__tablename__ = "threads_meta"
class ThreadMetaRow(Base):
__tablename__ = "threads_meta"
__table_args__ = (
Index("ix_threads_meta_created_user", "created_at", "user_id"),
Index("ix_threads_meta_user_created", "user_id", "created_at"),
)
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy import Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -21,6 +21,10 @@ class ToolCallMetricRow(Base):
"""
__tablename__ = "tool_call_metrics"
__table_args__ = (
Index("ix_tool_call_metrics_created_user", "created_at", "user_id"),
Index("ix_tool_call_metrics_run_created", "run_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
@@ -0,0 +1,18 @@
"""Per-user UI preferences persistence."""
from __future__ import annotations
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_preferences.sql import UserPreferencesRepository
__all__ = [
"UserPreferencesRepository",
"UserPreferencesRow",
"make_user_preferences_store",
]
def make_user_preferences_store(session_factory):
if session_factory is None:
return None
return UserPreferencesRepository(session_factory)
@@ -0,0 +1,26 @@
"""ORM model for per-user UI preferences.
A single key-value (JSON) row per user, holding small UI-level toggles that
should follow the account across devices — e.g. whether to hide the
admin-configured prompt prefix in the chat input. Kept generic (one JSON blob)
so new toggles do not each require an ALTER TABLE.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class UserPreferencesRow(Base):
__tablename__ = "user_preferences"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
preferences: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,41 @@
"""SQLAlchemy-backed per-user preferences repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.user_preferences.model import UserPreferencesRow
class UserPreferencesRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_for_user(self, user_id: str) -> dict[str, Any]:
"""Return the user's stored preferences (empty dict when none)."""
async with self._sf() as session:
row = await session.get(UserPreferencesRow, user_id)
if row is None or not isinstance(row.preferences, dict):
return {}
return dict(row.preferences)
async def merge_for_user(self, user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""Shallow-merge ``patch`` into the user's preferences and return the result."""
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.get(UserPreferencesRow, user_id)
if row is None:
merged = dict(patch)
row = UserPreferencesRow(user_id=user_id, preferences=merged, created_at=now, updated_at=now)
session.add(row)
else:
merged = dict(row.preferences or {})
merged.update(patch)
# Reassign so SQLAlchemy detects the change on the JSON column.
row.preferences = merged
row.updated_at = now
await session.commit()
return dict(merged)
@@ -19,5 +19,8 @@ class UserCustomPromptRow(Base):
title: Mapped[str] = mapped_column(String(255), nullable=False, default="")
content: Mapped[str] = mapped_column(Text, nullable=False)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Optional association to a single agent. NULL = global (shown everywhere);
# otherwise the prompt only surfaces when that agent is selected.
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -42,6 +42,7 @@ class UserCustomPromptRepository:
title=data.get("title") or "",
content=data["content"],
sort_order=data.get("sort_order", 0),
agent_id=(data.get("agent_id") or None),
created_at=now,
updated_at=now,
)
@@ -59,6 +60,10 @@ class UserCustomPromptRepository:
for key in ("title", "content", "sort_order"):
if key in data:
setattr(row, key, data[key])
# ``agent_id`` is explicitly nullable: an empty/None value clears the
# association (back to global), so accept None as a real update.
if "agent_id" in data:
row.agent_id = data["agent_id"] or None
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -21,6 +21,8 @@ import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import os
from pathlib import Path
from langgraph.types import Checkpointer
@@ -34,6 +36,110 @@ from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resol
logger = logging.getLogger(__name__)
def _csv_env(name: str) -> list[str]:
raw = os.getenv(name, "")
return [part.strip() for part in raw.split(",") if part.strip()]
def _dual_read_enabled(config) -> bool:
if getattr(config, "dual_read_enabled", False):
return True
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
def _use_postgres_enabled(config) -> bool:
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
if raw:
return raw.strip().lower() not in {"0", "false", "no", "off"}
return bool(getattr(config, "use_postgres", True))
def _dual_read_paths(config) -> list[str]:
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
resolved: list[str] = []
seen: set[str] = set()
for path in paths:
conn_str = resolve_sqlite_conn_str(path)
if conn_str == ":memory:" or conn_str.startswith("file:"):
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
continue
if conn_str not in seen:
resolved.append(conn_str)
seen.add(conn_str)
return resolved
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
configured = getattr(config, "sqlite_connection_string", None)
if configured:
return configured
paths = _dual_read_paths(config)
return paths[0] if paths else "store.db"
@contextlib.asynccontextmanager
async def _pg_pool_saver(conn_str: str) -> AsyncIterator[Checkpointer]:
"""Pool-backed ``AsyncPostgresSaver``, resilient to idle-closed connections.
``AsyncPostgresSaver.from_conn_string`` opens **one** long-lived connection.
对远程 Postgres(本项目 47.88.25.99 这类),空闲几分钟后连接会被服务端 / NAT /
连接池中间件断开,而 psycopg **不会自动重连** —— 下一次 checkpoint 操作就抛
``OperationalError: the connection is closed``,整个 run 直接失败(实测:圆桌第二步
完成后闲置 ~6 分钟,用户再人工干预 → leader run 读 checkpoint 时撞死连接 → 前端
"(无文本输出)")。
用带健康检查的连接池根治:``check=AsyncConnectionPool.check_connection`` 在把连接交给
调用方前先验活,坏连接直接丢弃重建;``max_idle`` 主动回收久闲连接,赶在远程 idle 超时
之前。这样上层永远拿不到失效连接。``prepare_threshold=0`` 关预备语句,兼容 pgbouncer
一类连接池(8432 非标准端口很可能就是)。
"""
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
from psycopg_pool import AsyncConnectionPool
try:
max_size = int(os.getenv("DEER_FLOW_PG_POOL_MAX", "20"))
except ValueError:
max_size = 20
pool = AsyncConnectionPool(
conninfo=conn_str,
min_size=1,
max_size=max_size,
open=False,
kwargs={"autocommit": True, "prepare_threshold": 0},
check=AsyncConnectionPool.check_connection,
max_idle=120.0,
max_lifetime=1800.0,
)
await pool.open(wait=True)
try:
saver = AsyncPostgresSaver(pool)
await saver.setup()
logger.info("Checkpointer: Postgres connection pool ready (max_size=%d, health-checked)", max_size)
yield saver
finally:
await pool.close()
@contextlib.asynccontextmanager
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(raw)
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
@@ -49,30 +155,64 @@ async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
return
if config.type == "sqlite":
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(config.connection_string or "store.db")
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
yield saver
return
if config.type == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not _use_postgres_enabled(config):
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
yield saver
return
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
await saver.setup()
yield saver
# 连接池 + 健康检查,根治远程 Postgres 单连接空闲被关导致的 run 崩溃(见 _pg_pool_saver)。
async with _pg_pool_saver(config.connection_string) as saver:
if not _dual_read_enabled(config):
yield saver
return
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.checkpointer.checkpoint_migration import (
DualReadCheckpointer,
SQLiteCheckpointSource,
)
sources: list[SQLiteCheckpointSource] = []
for sqlite_path in _dual_read_paths(config):
if not Path(sqlite_path).exists():
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
continue
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
source_saver = await ctx.__aenter__()
await source_saver.setup()
sources.append(
SQLiteCheckpointSource(
path=sqlite_path,
saver=source_saver,
context=ctx,
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
)
)
if not sources:
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
yield saver
return
proxy = DualReadCheckpointer(
saver,
sources,
)
try:
yield proxy
finally:
await proxy.aclose_sources()
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
@@ -106,16 +246,11 @@ async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpoi
return
if db_config.backend == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not db_config.postgres_url:
raise ValueError("database.postgres_url is required for the postgres backend")
async with AsyncPostgresSaver.from_conn_string(db_config.postgres_url) as saver:
await saver.setup()
# 连接池 + 健康检查(见 _pg_pool_saver),避免单连接空闲被关。
async with _pg_pool_saver(db_config.postgres_url) as saver:
yield saver
return
@@ -0,0 +1,276 @@
"""SQLite-to-Postgres checkpoint migration and read fallback helpers.
The live gateway can wrap a Postgres checkpointer with ``DualReadCheckpointer``.
New writes go to Postgres, reads prefer Postgres and fall back to legacy SQLite.
Manual migration scripts reuse the copy helpers in this module.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
@dataclass
class SQLiteCheckpointSource:
path: str
saver: Any
context: Any
progress_path: Path
closed: bool = False
async def close(self) -> None:
if self.closed:
return
self.closed = True
try:
await self.context.__aexit__(None, None, None)
except Exception:
logger.warning("Failed to close legacy SQLite checkpointer: %s", self.path, exc_info=True)
def _sqlite_db_exists(path: str) -> bool:
return Path(path).exists()
def _sqlite_threads(path: str) -> list[str]:
with sqlite3.connect(path) as conn:
rows = conn.execute("SELECT DISTINCT thread_id FROM checkpoints ORDER BY thread_id").fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_has_table(path: str, table: str) -> bool:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
(table,),
).fetchone()
return row is not None
def sqlite_recent_threads(path: str, *, since: str | None = None, until: str | None = None) -> list[str]:
"""Return threads updated since an ISO timestamp when ``threads_meta`` exists."""
if not _sqlite_has_table(path, "threads_meta"):
return []
query = "SELECT thread_id FROM threads_meta"
where: list[str] = []
params: list[str] = []
if since:
where.append("updated_at >= ?")
params.append(since)
if until:
where.append("updated_at <= ?")
params.append(until)
if where:
query += " WHERE " + " AND ".join(where)
query += " ORDER BY updated_at DESC"
with sqlite3.connect(path) as conn:
rows = conn.execute(query, tuple(params)).fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_thread_checkpoint_count(path: str, thread_id: str) -> int:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?",
(thread_id,),
).fetchone()
return int(row[0] if row else 0)
def _load_progress(path: Path) -> set[str]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
return set()
except Exception:
logger.warning("Ignoring unreadable checkpoint migration progress file: %s", path, exc_info=True)
return set()
migrated = payload.get("migrated_threads", [])
return {str(x) for x in migrated if x}
def _save_progress(path: Path, migrated: set[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(
json.dumps({"migrated_threads": sorted(migrated)}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
tmp.replace(path)
async def _copy_thread_state(source, target, thread_id: str) -> bool:
"""Copy one thread through the public LangGraph checkpointer API."""
config = {"configurable": {"thread_id": thread_id}}
tuples = [t async for t in source.alist(config)]
if not tuples:
return False
for tup in reversed(tuples):
configurable = (tup.config or {}).get("configurable", {}) or {}
checkpoint_ns = configurable.get("checkpoint_ns", "") or ""
parent_id = ((tup.parent_config or {}).get("configurable", {}) or {}).get("checkpoint_id")
put_config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": checkpoint_ns}}
if parent_id:
put_config["configurable"]["checkpoint_id"] = parent_id
new_versions = (tup.checkpoint or {}).get("channel_versions", {}) or {}
await target.aput(put_config, tup.checkpoint, tup.metadata or {}, new_versions)
if tup.pending_writes:
by_task: dict[str, list] = {}
for task_id, channel, value in tup.pending_writes:
by_task.setdefault(task_id, []).append((channel, value))
checkpoint_id = (tup.checkpoint or {}).get("id")
for task_id, writes in by_task.items():
writes_config = {
"configurable": {
"thread_id": thread_id,
"checkpoint_ns": checkpoint_ns,
"checkpoint_id": checkpoint_id,
}
}
await target.aput_writes(writes_config, writes, task_id)
return True
async def copy_thread_state(source, target, thread_id: str) -> bool:
return await _copy_thread_state(source, target, thread_id)
async def _target_thread_checkpoint_count(target, thread_id: str) -> int:
config = {"configurable": {"thread_id": thread_id}}
count = 0
async for _ in target.alist(config):
count += 1
return count
async def target_has_thread(target, thread_id: str) -> bool:
config = {"configurable": {"thread_id": thread_id}}
async for _ in target.alist(config, limit=1):
return True
return False
class DualReadCheckpointer:
"""Postgres-first checkpointer with SQLite read fallback."""
def __init__(self, primary, sources: list[SQLiteCheckpointSource], *, sleep_seconds: float = 0.05):
self.primary = primary
self.sources = sources
self.sleep_seconds = max(0.0, sleep_seconds)
self._closed = False
def __getattr__(self, name: str):
return getattr(self.primary, name)
async def aclose_sources(self) -> None:
self._closed = True
for source in self.sources:
await source.close()
async def aget_tuple(self, config):
found = await self.primary.aget_tuple(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget_tuple(config)
if found is not None:
return found
return None
async def aget(self, config):
found = await self.primary.aget(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget(config)
if found is not None:
return found
return None
async def alist(self, config, *, filter=None, before=None, limit=None):
seen: set[tuple[str, str, str]] = set()
yielded = 0
async for tup in self.primary.alist(config, filter=filter, before=before, limit=limit):
key = self._tuple_key(tup)
seen.add(key)
yield tup
yielded += 1
if limit is not None and yielded >= limit:
return
remaining = None if limit is None else max(0, limit - yielded)
for source in self.sources:
if source.closed:
continue
async for tup in source.saver.alist(config, filter=filter, before=before, limit=remaining):
key = self._tuple_key(tup)
if key in seen:
continue
seen.add(key)
yield tup
yielded += 1
if limit is not None:
remaining = max(0, limit - yielded)
if remaining <= 0:
return
async def aput(self, *args, **kwargs):
return await self.primary.aput(*args, **kwargs)
async def aput_writes(self, *args, **kwargs):
return await self.primary.aput_writes(*args, **kwargs)
async def adelete_thread(self, thread_id: str) -> None:
delete = getattr(self.primary, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
for source in self.sources:
if source.closed:
continue
delete = getattr(source.saver, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
@staticmethod
def _tuple_key(tup) -> tuple[str, str, str]:
configurable = (tup.config or {}).get("configurable", {}) or {}
return (
str(configurable.get("thread_id", "")),
str(configurable.get("checkpoint_ns", "") or ""),
str(configurable.get("checkpoint_id", "") or (tup.checkpoint or {}).get("id", "")),
)
async def _verify_source(self, source: SQLiteCheckpointSource, threads: list[str]) -> bool:
for thread_id in threads:
source_count = await asyncio.to_thread(_sqlite_thread_checkpoint_count, source.path, thread_id)
target_count = await _target_thread_checkpoint_count(self.primary, thread_id)
if target_count < source_count:
logger.warning(
"Checkpoint migration verification failed: source=%s thread_id=%s source_count=%d target_count=%d",
source.path,
thread_id,
source_count,
target_count,
)
return False
return True
@@ -93,3 +93,12 @@ class RunStore(abc.ABC):
by_caller ({lead_agent, subagent, middleware}).
"""
pass
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` across all runs.
One run corresponds to one user submission, so this backs the admin
user center's 问答次数 (Q&A count) column — each question counts once.
Non-overriding stores return an empty mapping (treated as 0 per user).
"""
return {}
@@ -38,6 +38,11 @@ _DANGEROUS_URL_RE = re.compile(r"^\s*(javascript|vbscript|data:text/html)", re.I
# Attributes carrying a URL we must scheme-check.
_URL_ATTRS = frozenset({"href", "src", "action", "formaction", "xlink:href", "background", "poster"})
_LOCAL_DOC_LINK_RE = re.compile(
r"^(?![a-z][a-z0-9+.-]*:)(?!//)(?P<path>[^#?]*\.(?:html?|md|markdown))(?P<query>\?[^#]*)?(?P<fragment>#.+)?$",
re.IGNORECASE,
)
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
"""Split a ``---`` YAML frontmatter block from the body of a SKILL.md.
@@ -225,6 +230,38 @@ def sanitize_html(html: str) -> tuple[str, list[str]]:
return str(soup), removed
def constrain_same_page_anchor_links(html: str) -> tuple[str, list[str]]:
"""Keep generated HTML document jumps inside the current HTML file."""
if not html or not isinstance(html, str):
return "", []
try:
from bs4 import BeautifulSoup
except Exception: # pragma: no cover - bs4 is a declared dependency
logger.warning("BeautifulSoup unavailable; returning HTML links unconstrained")
return html, []
changed: list[str] = []
soup = BeautifulSoup(html, "html.parser")
for tag in soup.find_all("a"):
href = tag.get("href")
if not isinstance(href, str):
continue
value = href.strip()
if not value or value.startswith("#"):
continue
match = _LOCAL_DOC_LINK_RE.match(value)
if not match:
continue
fragment = match.group("fragment")
if fragment:
tag["href"] = fragment
changed.append(f"{value}->{fragment}")
else:
del tag.attrs["href"]
changed.append(f"{value}->removed")
return str(soup), changed
def validate_html(html: str) -> list[str]:
"""Return a list of structural problems with ``html`` (empty == valid).
@@ -299,6 +336,7 @@ _GENERATION_TEMPLATE = """你是一个内网页面生成助手。请根据输入
6. 不要包含 <script>、外链脚本、iframe、表单提交、on* 事件属性。
7. 参考页面只用于风格和布局,页面内容必须来自“本次数据”,不要复用参考页旧数据。
8. 页面使用中文,排版适合在浏览器中直接预览。
9. HTML 内的目录/锚点跳转必须只跳转到当前这个 HTML 文件内部:只使用 href="#section-id",不要使用 href="index.html#section-id"、".md#..."、其它文件路径或任何跨文件锚点;目标区块必须有匹配的 id。
"""
_REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问题,请修复后重新输出完整页面。
@@ -317,6 +355,7 @@ _REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问
2. fixed_html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
3. 移除所有 <script>、iframe、外链脚本、on* 事件属性、表单提交。
4. 保留原有内容与数据,只修复结构与安全问题。
5. HTML 内的目录/锚点跳转必须只跳转到当前这个 HTML 文件内部:只使用 href="#section-id",不要链接到 .html/.md 文件作为锚点。
"""
_STYLE_EXTRACT_TEMPLATE = """你是一个页面风格分析助手。请分析下面的 HTML 页面,提取可复用的风格与布局特征。
@@ -53,8 +53,18 @@ _MARKDOWN_MIN_CHARS = 100
_MARKDOWN_INSTRUCTION = (
"\n\n【输出要求】请将本次任务的最终结果生成为一个 Markdown(.md) 格式的文件,"
"保存到 /mnt/user-data/outputs 目录,并使用 present_files 展示该文件。"
"Markdown 内的目录/锚点跳转必须只跳转到当前这个 Markdown 文件内部:"
"只使用 [标题](#section-id),不要使用 [标题](xxx.md#section-id)、"
"[标题](xxx.html#section-id) 或任何跨文件锚点。"
)
_LOCAL_DOC_MARKDOWN_LINK_RE = re.compile(
r"^(?![a-z][a-z0-9+.-]*:)(?!//)(?P<path>[^#?]*\.(?:html?|md|markdown))(?P<query>\?[^#]*)?(?P<fragment>#.+)?$",
re.IGNORECASE,
)
_MARKDOWN_INLINE_LINK_RE = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
_MARKDOWN_REFERENCE_LINK_RE = re.compile(r"^(\s*\[[^\]]+\]:\s*)(\S+)(.*)$", re.MULTILINE)
class MarkdownRequirementError(Exception):
"""Raised when a ``require_markdown`` task did not produce a valid .md file.
@@ -219,6 +229,23 @@ def _task_requires_markdown(task: dict[str, Any]) -> bool:
return bool(isinstance(ctx, dict) and ctx.get("require_markdown"))
def _task_requires_html(task: dict[str, Any]) -> bool:
"""Whether a task must produce an HTML page (the ``必须生成 HTML`` switch).
The flag lives inside ``execution_context.require_html``. For backward
compatibility, tasks created under the old two-tab UI (which stored
``task_kind == "html_page"``) are still treated as HTML tasks, so no data
migration is needed. ``require_html`` and ``require_markdown`` are
independent — both may be on at once.
"""
ctx = task.get("execution_context")
if not isinstance(ctx, dict):
return False
if ctx.get("require_html"):
return True
return ctx.get("task_kind") == "html_page"
def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
"""True when the run produced a ``.md`` file whose text exceeds the minimum.
@@ -245,6 +272,43 @@ def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
return False
def _normalize_same_page_markdown_target(target: str) -> str | None:
value = (target or "").strip()
if not value or value.startswith("#"):
return value
match = _LOCAL_DOC_MARKDOWN_LINK_RE.match(value)
if not match:
return value
fragment = match.group("fragment")
return fragment or None
def _constrain_markdown_same_page_anchor_links(content: str) -> str:
"""Keep generated Markdown document jumps inside the current Markdown file."""
if not content or not isinstance(content, str):
return content or ""
def replace_inline(match: re.Match[str]) -> str:
label = match.group(1)
target = match.group(2)
normalized = _normalize_same_page_markdown_target(target)
if normalized is None:
return label
return f"[{label}]({normalized})"
def replace_reference(match: re.Match[str]) -> str:
prefix = match.group(1)
target = match.group(2)
suffix = match.group(3)
normalized = _normalize_same_page_markdown_target(target)
if normalized is None:
return f"{prefix}#"
return f"{prefix}{normalized}{suffix}"
content = _MARKDOWN_INLINE_LINK_RE.sub(replace_inline, content)
return _MARKDOWN_REFERENCE_LINK_RE.sub(replace_reference, content)
def _build_result(
task: dict[str, Any],
*,
@@ -905,15 +969,18 @@ class ScheduledTaskService:
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Dispatch a single execution attempt by task kind.
"""Dispatch a single execution attempt by the task's output requirements.
``html_page`` tasks render an HTML document via GLM5; everything else
runs the regular lead-agent attempt. The kind lives in
``execution_context.task_kind`` so no schema migration is needed.
A task carries two independent switches in ``execution_context``:
``require_html`` (必须生成 HTML) and ``require_markdown`` (必须生成
Markdown). When ``require_html`` is set the run goes through the HTML
render pipeline (which also enforces the Markdown requirement when both
switches are on); otherwise it runs the regular lead-agent attempt.
Both flags live in ``execution_context`` so no schema migration is
needed, and old ``task_kind == "html_page"`` tasks still resolve to the
HTML pipeline via :func:`_task_requires_html`.
"""
ctx = task.get("execution_context")
kind = ctx.get("task_kind") if isinstance(ctx, dict) else None
if kind == "html_page":
if _task_requires_html(task):
return await self._run_html_page_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
@@ -956,17 +1023,21 @@ class ScheduledTaskService:
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> str:
) -> tuple[str, list[dict[str, Any]]]:
"""Run the lead agent to collect source data for the HTML page.
Reuses the regular agent-run primitives (with the task's search query
folded into the prompt) and returns the agent's final text, which feeds
the GLM5 render step as ``本次数据``. Returns an empty string if the
agent produced nothing.
folded into the prompt) and returns ``(text, files)`` — the agent's
final text (feeds the render step as ``本次数据``) plus any files it
produced. The files are only captured/needed when the task also has the
Markdown switch on (both 必须生成 HTML and 必须生成 Markdown), so the run
can validate and surface the ``.md`` file alongside the HTML page;
otherwise the list is empty.
"""
cfg = self._html_config(task)
page_prompt = str(cfg.get("page_prompt") or "").strip()
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
also_markdown = _task_requires_markdown(task)
gather_instruction = (
"请运用你自身的检索/搜索等技能,围绕下述页面需求获取最新、真实的资料,"
"并整理成结构化的中文要点(可用列表/小标题),供后续生成 HTML 页面使用,本步骤不要生成 HTML:\n"
@@ -982,12 +1053,22 @@ class ScheduledTaskService:
# which sources/data the agent collects, otherwise it silently leaks in.
if negative_prompt:
parts.append(f"限制条件(最高优先级,必须严格遵守):{negative_prompt}")
gather_task = {**task, "prompt": "\n".join(parts)}
# Never apply the markdown gate to the gather pass.
gather_prompt = "\n".join(parts)
# When the Markdown switch is also on, this same gather pass must save a
# .md file. We do NOT append _MARKDOWN_INSTRUCTION here — `_start_agent_run`
# already appends it exactly once when `require_markdown` is present in the
# execution context (kept below). Appending it here too made the prompt
# carry the 【输出要求】 block twice when both HTML and Markdown were on.
gather_task = {**task, "prompt": gather_prompt}
gather_ctx = dict(task.get("execution_context") or {})
gather_ctx.pop("require_markdown", None)
# Strip the HTML-specific flags so the gather pass is a plain agent run.
gather_ctx.pop("require_html", None)
gather_ctx.pop("task_kind", None)
gather_ctx.pop("html_config", None)
# Keep ``require_markdown`` only when both switches are on, so the agent
# is told (via _start_agent_run) to produce the .md file.
if not also_markdown:
gather_ctx.pop("require_markdown", None)
# Use the task's selected model for the data-gathering agent too, so the
# whole HTML pipeline honors ``html_config.model_name`` instead of
# silently falling back to the default model for this step.
@@ -997,13 +1078,22 @@ class ScheduledTaskService:
gather_task["execution_context"] = gather_ctx
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
files: list[dict[str, Any]] = []
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"]) if also_markdown else None
record = await self._start_agent_run(gather_task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
text = await self._extract_latest_ai_text(runtime_thread_id)
return text or ""
if also_markdown:
files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_id,
output_snapshot=output_snapshot,
)
return text or "", files
def _collect_html_skill_instructions(self) -> str:
"""Concatenate the bodies of enabled skills marked for HTML pages.
@@ -1072,10 +1162,17 @@ class ScheduledTaskService:
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
reference_favorite_id = cfg.get("reference_favorite_id") or None
# 1) Gather source data (real, via the agent's search tools).
data_text = await self._gather_html_data(
# 1) Gather source data (real, via the agent's search tools). When the
# Markdown switch is also on, the gather pass additionally saves the .md
# file, returned here so we can validate + surface it next to the page.
also_markdown = _task_requires_markdown(task)
data_text, gathered_files = await self._gather_html_data(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
if also_markdown and not _has_valid_markdown_file(gathered_files):
raise MarkdownRequirementError(
f"未满足输出要求:需生成 Markdown(.md) 文件,且文件正文不少于 {_MARKDOWN_MIN_CHARS} 字"
)
# 2) Load reference style/layout from a favorite if one was selected.
reference_style = None
@@ -1119,6 +1216,7 @@ class ScheduledTaskService:
# 4) Sanitize + validate, with one repair pass on failure.
html, _removed = html_page_utils.sanitize_html(html)
html, _link_changes = html_page_utils.constrain_same_page_anchor_links(html)
errors = html_page_utils.validate_html(html)
if errors:
logger.info("Generated HTML failed validation (%s); attempting repair", errors)
@@ -1131,6 +1229,7 @@ class ScheduledTaskService:
fixed = repaired.get("fixed_html")
if isinstance(fixed, str) and "<" in fixed:
fixed_clean, _ = html_page_utils.sanitize_html(fixed)
fixed_clean, _ = html_page_utils.constrain_same_page_anchor_links(fixed_clean)
if not html_page_utils.validate_html(fixed_clean):
html = fixed_clean
except Exception:
@@ -1158,6 +1257,13 @@ class ScheduledTaskService:
archive_path = self._archive_html_index(run_id, html)
files: list[dict[str, Any]] = []
# When both switches are on, surface the gathered Markdown file(s)
# alongside the generated page so the run delivers both artifacts.
if also_markdown:
for item in gathered_files:
name = str(item.get("name") or "") if isinstance(item, dict) else ""
if name.lower().endswith((".md", ".markdown")):
files.append(item)
if archive_path is not None:
files.append(
{
@@ -1483,6 +1589,14 @@ class ScheduledTaskService:
index += 1
shutil.copy2(path, candidate)
if candidate.suffix.lower() in {".md", ".markdown"}:
try:
original = candidate.read_text(encoding="utf-8", errors="ignore")
constrained = _constrain_markdown_same_page_anchor_links(original)
if constrained != original:
candidate.write_text(constrained, encoding="utf-8", newline="")
except Exception:
logger.debug("Failed to constrain scheduled markdown links in %s", candidate, exc_info=True)
return candidate
def _build_file_result(self, *, virtual_path: str, source_path: Path, archive_path: Path) -> dict[str, Any]:
@@ -36,6 +36,7 @@ _DEFAULT_ENTRY: dict = {
"created_at": None,
"state": STATE_ACTIVE,
"pinned": False,
"featured": False,
"archived_at": None,
"source": SOURCE_UPLOAD,
}
@@ -141,6 +142,11 @@ def set_pinned(name: str, pinned: bool) -> None:
_update_entry(name, {"pinned": pinned})
def set_featured(name: str, featured: bool) -> None:
"""置顶/取消置顶。与 pinned(防归档)正交,仅影响列表排序展示。"""
_update_entry(name, {"featured": featured})
def forget(name: str) -> None:
try:
with _usage_lock:
@@ -0,0 +1,82 @@
"""read_peer_delivery —— 圆桌席位「按需读取前序席位**完整交付**」的只读工具。
背景:圆桌每个席位是独立 thread(独立 sandbox),彼此隔离。默认情况下,一个席位只能
从跨席位广播里看到前序席位的**截断摘要**(≤1500 字)。当席位识别到自己需要某个前序
席位的**全文**时,调用本工具按席位名取回完整交付。
数据来源:全文由**网关层**(``app.gateway``)在发起本席位 run 时,注入到
``runtime.context["roundtable_peer_deliveries"]`` —— 一个
``[{"name": <席位显示名>, "content": <完整交付正文>}, ...]`` 列表。本工具**只读该上下文
dict**,不跨 thread、不碰 sandbox、不发网络请求,因此完全在 ``deerflow.*`` 内(不违反
harness→app 导入防火墙)。非圆桌场景下该上下文为空,工具返回「无」。
「传摘要按需传全文」:完整交付**只在席位真正调用本工具时**才进入它的 LLM 上下文,平时
只有摘要,既省 token 又满足「需要时能拿全文」。详见
``frontend-web/docs/roundtable-dag-orchestration.md`` §12.9。
"""
from __future__ import annotations
import logging
from langchain.tools import ToolRuntime, tool
logger = logging.getLogger(__name__)
_CTX_KEY = "roundtable_peer_deliveries"
def _peer_deliveries(runtime: ToolRuntime | None) -> list[dict]:
"""从 runtime.context 取出网关注入的前序席位完整交付列表(缺失/非法时返回 [])。"""
if runtime is None:
return []
ctx = getattr(runtime, "context", None)
if not ctx:
return []
try:
peers = ctx.get(_CTX_KEY)
except Exception:
return []
if not isinstance(peers, list):
return []
return [p for p in peers if isinstance(p, dict)]
@tool("read_peer_delivery", parse_docstring=True)
def read_peer_delivery_tool(runtime: ToolRuntime, seat_name: str = "") -> str:
"""Read another roundtable seat's FULL deliverable on demand.
In a multi-agent roundtable you (a seat) normally only see **truncated
summaries** of earlier seats' deliverables. When you decide you genuinely
need the **complete text** of a specific earlier seat (e.g. to build on its
detailed data/conclusions), call this tool with that seat's name to fetch
its full deliverable. Prefer the summaries you already have; only call this
when the summary is insufficient.
Args:
seat_name: The earlier seat's display name (the Chinese name shown in
the summaries, e.g. "情报收集"). Leave empty to list which peers'
full deliverables are available to read.
"""
peers = _peer_deliveries(runtime)
if not peers:
return "(当前没有可读取的前序席位完整交付。请直接基于你已有的上下文与摘要完成交付。)"
names = [str(p.get("name") or "") for p in peers if p.get("name")]
query = (seat_name or "").strip()
if not query:
return (
"可读取**完整交付**的前序席位:" + "、".join(names) + "。\n"
"需要哪个席位的全文,就再次调用 read_peer_delivery 并把它的名称传给 seat_name。"
)
ql = query.lower()
# 精确匹配(忽略大小写)优先,其次子串互含(容忍模型给「情报收集席位」这类带后缀的名)。
for p in peers:
if str(p.get("name") or "").lower() == ql:
return str(p.get("content") or "(该席位交付为空。)")
for p in peers:
nm = str(p.get("name") or "")
if nm and (ql in nm.lower() or nm.lower() in ql):
return str(p.get("content") or "(该席位交付为空。)")
return f"未找到名为「{query}」的前序席位。可读取的有:" + "、".join(names) + "。"
@@ -114,6 +114,13 @@ def get_available_tools(
builtin_tools.extend([skill_list_tool, skill_view_tool])
# read_peer_delivery —— 圆桌席位「按需读取前序席位完整交付」的只读工具(从 run context
# 取网关注入的全文,非圆桌场景为空 → 工具自述「无」)。始终提供(只读、无副作用);圆桌
# 总控/报告角色由 run policy 显式排除,避免干扰它们。
from deerflow.tools.builtins.roundtable_peers_tool import read_peer_delivery_tool
builtin_tools.append(read_peer_delivery_tool)
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
# Gateway 每次 run 重建 agent,用户改了配置下次 run 即生效;解析失败退回全局 config.yaml。
@@ -31,9 +31,12 @@ dependencies = [
"ddgs>=9.10.0",
"duckdb>=1.4.4",
"langchain-google-genai>=4.2.1",
"langgraph-checkpoint-sqlite>=3.0.3",
"langgraph-sdk>=0.1.51",
"sqlalchemy[asyncio]>=2.0,<3.0",
"langgraph-checkpoint-sqlite>=3.0.3",
"langgraph-checkpoint-postgres>=3.0.5",
"langgraph-sdk>=0.1.51",
"psycopg[binary]>=3.3.3",
"psycopg-pool>=3.3.0",
"sqlalchemy[asyncio]>=2.0,<3.0",
"aiosqlite>=0.19",
"alembic>=1.13",
]