This commit is contained in:
许海洋 committed 2026-05-20 17:43:50 +08:00
commit e2d61f4cfb
476 files changed
+103681 -9379

No files matched your search

@@ -0,0 +1,15 @@
from .graph import (
AI_WRITING_GRAPH,
build_ai_writing_graph,
get_ai_writing_checkpointer,
get_ai_writing_graph,
shutdown_ai_writing_graph,
)
__all__ = [
"AI_WRITING_GRAPH",
"build_ai_writing_graph",
"get_ai_writing_graph",
"get_ai_writing_checkpointer",
"shutdown_ai_writing_graph",
]
@@ -0,0 +1,538 @@
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
from __future__ import annotations
import logging
from langgraph.graph import END, StateGraph
from langgraph.types import interrupt
from .state import AIWritingState
from .nodes.intent_parser import intent_parser_node
from .nodes.researcher import researcher_node
from .nodes.writer_outline import writer_outline_node
from .nodes.writer_draft import writer_draft_node
from .nodes.editor import editor_node
logger = logging.getLogger(__name__)
# ── 干预节点 ──────────────────────────────────────────────────────────────────
async def pause_material_confirm(state: AIWritingState) -> dict:
"""PAUSE-1:素材确认"""
pkg = state["material_package"]
user_input: dict = interrupt({
"pause_point": "material_confirm",
"message": "素材收集专家已完成素材搜集,请确认素材后继续。",
"materials": pkg["materials"],
"keywords": pkg["keywords"],
"summary": pkg["summary"],
})
action = user_input.get("action", "confirm")
if action == "confirm":
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
updated_pkg = {**pkg, "materials": filtered}
return {
"material_package": updated_pkg,
"last_user_intervention": user_input,
"status": "writing_outline",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材,开始规划大纲"}],
}
# re_search:重新搜索
extra_kws = user_input.get("extraKeywords") or []
return {
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
"status": "researching",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
}
def _normalize_outline(outline: dict) -> dict:
"""将前端回传的 camelCase 大纲统一为后端 snake_case 格式。"""
if not outline:
return outline
sections = []
for s in outline.get("sections", []):
sections.append({
"section_title": s.get("section_title") or s.get("sectionTitle", ""),
"key_points": s.get("key_points") or s.get("keyPoints", []),
"material_ids": s.get("material_ids") or s.get("materialIds", []),
})
return {"title": outline.get("title", ""), "sections": sections}
def _check_outline_material_support(outline: dict) -> str:
"""轻量启发式:统计未关联任何素材的章节,返回预警文案(空串=无需预警)。
仅做提示、不拦截流程;真正的素材不足会在写作节点被检出并触发求助暂停。
"""
sections = (outline or {}).get("sections", [])
weak = [
(s.get("section_title") or s.get("sectionTitle") or "")
for s in sections
if not (s.get("material_ids") or s.get("materialIds"))
]
weak = [t for t in weak if t]
if not weak:
return ""
return (
f"有 {len(weak)} 个章节未关联检索素材({('、'.join(weak))})。"
"严格依据素材模式下,这些章节在写作时可能因素材不足而暂停向你求助。"
)
async def pause_outline_confirm(state: AIWritingState) -> dict:
"""PAUSE-2:大纲确认"""
user_input: dict = interrupt({
"pause_point": "outline_confirm",
"message": "作家已完成大纲规划,请确认或编辑大纲后继续。",
"outline": state["current_outline"],
})
action = user_input.get("action", "confirm")
if action == "confirm":
raw_outline = user_input.get("editedOutline") or state["current_outline"]
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
events: list[dict] = [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}]
# 阶段四:严格模式下对大纲做轻量素材充分性预警(仅提示,不拦截流程)
if state.get("writing_mode") == "strict":
warning = _check_outline_material_support(edited)
if warning:
events.append({
"type": "notice", "agent_name": "系统",
"title": "素材充分性提示", "detail": warning,
})
return {
"current_outline": edited,
"last_user_intervention": user_input,
"status": "writing_draft",
"progress_events": events,
}
# re_outline
feedback = user_input.get("outlineFeedback", "")
updates: dict = {
"last_user_intervention": {**user_input, "_feedback": feedback},
"status": "writing_outline",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
}
# 用户在卡片里手动改过大纲 → 以手动版作为本次重规划的编辑基准
raw_edited = user_input.get("editedOutline")
if raw_edited:
updates["current_outline"] = _normalize_outline(raw_edited)
return updates
async def pause_draft_confirm(state: AIWritingState) -> dict:
"""PAUSE-3:草稿确认"""
draft = state["current_draft"]
user_input: dict = interrupt({
"pause_point": "draft_confirm",
"message": "草稿已完成,请确认或给出修改意见。",
"draft_title": draft["title"],
"revision_count": draft["revision_count"],
"review_result": state.get("review_result"),
})
action = user_input.get("action", "to_editor")
if action == "finalize":
return {
"last_user_intervention": user_input,
"status": "done",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户确认定稿"}],
}
if action == "user_revise":
notes = user_input.get("userRevisionNotes", "")
return {
"last_user_intervention": {**user_input, "_user_notes": notes},
"revision_count": state.get("revision_count", 0) + 1,
"status": "revising",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"按用户意见重写({notes[:30]})"}],
}
# to_editor
return {
"last_user_intervention": user_input,
"status": "reviewing",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "提交编辑审核"}],
}
async def pause_section_help(state: AIWritingState) -> dict:
"""素材不足求助点:严格模式下作家发现章节缺素材,让用户逐章决定如何处理。
用户对每个被阻塞章节选择:
- supplement:补充检索素材(整体回到检索流程重走)
- delete:删除该章节
- loose:按通用知识续写该章节
"""
blocked = state.get("blocked_sections") or []
user_input: dict = interrupt({
"pause_point": "section_help",
"message": "部分章节缺少足够素材支撑写作,请决定如何处理。",
"blocked_sections": blocked,
})
# sectionDecisions: [{"sectionTitle": "...", "decision": "supplement|delete|loose"}]
by_title: dict = {}
for d in user_input.get("sectionDecisions") or []:
title = d.get("sectionTitle") or d.get("section_title")
if title:
by_title[title] = d.get("decision", "loose")
# 任一章节选「补充检索」→ 整体回到检索,重走「检索 → 素材确认 → 大纲 → 写作」
supplement_reasons = [
b["reason"] for b in blocked if by_title.get(b["section_title"]) == "supplement"
]
if supplement_reasons:
return {
# supplement_search:补充检索(保留已有素材、追加新素材),完成后直接回写作、不重排大纲
"last_user_intervention": {**user_input, "action": "supplement_search", "_extra_keywords": supplement_reasons},
"status": "researching",
"blocked_sections": None,
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "补充检索素材中…"}],
}
# 否则:删除标记章节、记录「通用知识续写」章节,直接回写作
outline = state.get("current_outline") or {"title": "", "sections": []}
kept, loose_titles = [], []
for s in outline.get("sections", []):
title = s.get("section_title") or s.get("sectionTitle", "")
decision = by_title.get(title)
if decision == "delete":
continue
if decision == "loose":
loose_titles.append(title)
kept.append(s)
# 防御:用户把所有章节都删了 → 保留原大纲并全部按通用知识写,避免空文章
if not kept:
kept = outline.get("sections", [])
loose_titles = [s.get("section_title") or s.get("sectionTitle", "") for s in kept]
return {
"current_outline": {**outline, "sections": kept},
"last_user_intervention": {**user_input, "_loose_sections": loose_titles},
"status": "writing_draft",
"blocked_sections": None,
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "已按你的选择继续写作…"}],
}
async def pause_review_confirm(state: AIWritingState) -> dict:
"""PAUSE-4:编辑打回确认"""
user_input: dict = interrupt({
"pause_point": "review_confirm",
"message": "编辑认为文章需要修改,请决定下一步操作。",
"review_result": state["review_result"],
"revision_count": state.get("revision_count", 0),
"max_revisions": state.get("max_revisions", 3),
})
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
return {
"last_user_intervention": user_input,
"status": "done",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户强制定稿"}],
}
action = user_input.get("action", "accept_review")
extra_notes = user_input.get("userRevisionNotes", "")
return {
"last_user_intervention": {**user_input, "_extra_notes": extra_notes, "action": action},
"revision_count": state.get("revision_count", 0) + 1,
"status": "revising",
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "按编辑意见重写"}],
}
# ── 条件路由 ──────────────────────────────────────────────────────────────────
def after_researcher(state: AIWritingState) -> str:
"""检索完成后:补充检索(素材不足求助)直接回写作,否则去素材确认。"""
return "writer_draft" if state.get("status") == "writing_draft" else "pause_material"
def after_material_pause(state: AIWritingState) -> str:
return "researcher" if state["status"] == "researching" else "writer_outline"
def after_outline_pause(state: AIWritingState) -> str:
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
def after_writer_draft(state: AIWritingState) -> str:
"""写作完成后:素材不足则去求助暂停点,否则去草稿确认。"""
return "pause_section_help" if state["status"] == "awaiting_section_help" else "pause_draft"
def after_section_help_pause(state: AIWritingState) -> str:
"""求助暂停点之后:补充检索回 researcher,否则回写作。"""
return "researcher" if state["status"] == "researching" else "writer_draft"
def after_draft_pause(state: AIWritingState) -> str:
status = state["status"]
if status == "done":
return "end"
if status == "revising":
return "writer_draft"
return "editor"
def after_review(state: AIWritingState) -> str:
review = state.get("review_result") or {}
if review.get("verdict") == "pass":
return "pause_draft"
rev_count = state.get("revision_count", 0)
max_rev = state.get("max_revisions", 3)
if rev_count >= max_rev:
return "pause_draft" # 超出上限,强制到草稿确认
return "pause_review"
def after_review_pause(state: AIWritingState) -> str:
return "end" if state["status"] == "done" else "writer_draft"
# ── 组装 Graph ────────────────────────────────────────────────────────────────
def build_ai_writing_graph(checkpointer=None):
"""构建并 compile AI 写作 graph。
Args:
checkpointer: 可选的 LangGraph checkpointer。
- ``None`` → 退回 ``MemorySaver`` —— 仅供测试 / 开发使用。
进程或容器重启后会话状态全部丢失,前端拿着旧 session_id 调
``/resume`` 会得到 404。
- 其他 → 传入的 (Async)BaseCheckpointSaver 实例,用于持久化。
"""
g = StateGraph(AIWritingState)
g.add_node("intent_parser", intent_parser_node)
g.add_node("researcher", researcher_node)
g.add_node("pause_material", pause_material_confirm)
g.add_node("writer_outline", writer_outline_node)
g.add_node("pause_outline", pause_outline_confirm)
g.add_node("writer_draft", writer_draft_node)
g.add_node("pause_section_help", pause_section_help)
g.add_node("pause_draft", pause_draft_confirm)
g.add_node("editor", editor_node)
g.add_node("pause_review", pause_review_confirm)
g.set_entry_point("intent_parser")
g.add_edge("intent_parser", "researcher")
g.add_conditional_edges("researcher", after_researcher,
{"pause_material": "pause_material", "writer_draft": "writer_draft"})
g.add_conditional_edges("pause_material", after_material_pause,
{"researcher": "researcher", "writer_outline": "writer_outline"})
g.add_edge("writer_outline", "pause_outline")
g.add_conditional_edges("pause_outline", after_outline_pause,
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
g.add_conditional_edges("writer_draft", after_writer_draft,
{"pause_section_help": "pause_section_help", "pause_draft": "pause_draft"})
g.add_conditional_edges("pause_section_help", after_section_help_pause,
{"researcher": "researcher", "writer_draft": "writer_draft"})
g.add_conditional_edges("pause_draft", after_draft_pause,
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
g.add_conditional_edges("editor", after_review,
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
g.add_conditional_edges("pause_review", after_review_pause,
{"end": END, "writer_draft": "writer_draft"})
if checkpointer is None:
# MemorySaver 仅作为 fallback —— 提示一下,正式部署应通过
# get_ai_writing_graph() 拿到带持久化 checkpointer 的 graph。
from langgraph.checkpoint.memory import MemorySaver
checkpointer = MemorySaver()
return g.compile(checkpointer=checkpointer)
# ── 持久化 checkpointer 单例(懒加载)─────────────────────────────────────────
# 模块级 graph 槽位。import 时为 None;首次 ``get_ai_writing_graph()`` 调用时
# 按 config.yaml 的 checkpointer 段懒构建出真正的 graph。
# 测试可通过 ``monkeypatch.setattr(graph_mod, "AI_WRITING_GRAPH", stub)``
# 直接注入桩;本模块所有访问点都走 ``get_ai_writing_graph()`` 这一入口,会优先
# 返回 module attribute 的值,保留测试桩注入路径。
AI_WRITING_GRAPH = None
# 持有 async checkpointer(底层 aiosqlite Connection 或 psycopg pool 的所有者)。
# 进程生命周期内常驻;shutdown 时由 ``shutdown_ai_writing_graph`` 显式关闭。
_async_checkpointer_singleton = None
def _fallback_to_memory_saver(reason: str):
"""退回 InMemorySaver 的统一兜底入口。
把 reason 打到 warning 日志里方便运维定位。InMemorySaver 进程重启会丢
状态,但**当次写作能跑完** —— 我们的首要目标是「写作流程能继续」,
持久化是锦上添花。
"""
from langgraph.checkpoint.memory import InMemorySaver
logger.warning(
"ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver。原因: %s",
reason,
)
return InMemorySaver()
async def _get_async_checkpointer():
"""按 config.yaml 的 checkpointer 段返回一个 async checkpointer 实例。
**绝不抛异常** —— 任何失败一律退回 InMemorySaver。这是为了保证「写作
流程能继续」这个首要目标:持久化坏了用户至少还能写完当次稿,重启会丢
会话但不影响当下创作。
优先级:
1. ``checkpointer:`` 段(``type=sqlite|postgres|memory``)
2. fallback 到 ``InMemorySaver`` 并打 warning
返回的 saver 与主 agent 公用同一 sqlite 文件 —— LangGraph 的 schema
用 ``(thread_id, checkpoint_ns)`` 做主键,AI 写作的 thread_id 是
会话 uuid,跟主 agent 的 LangGraph thread_id 不会撞。
"""
global _async_checkpointer_singleton
if _async_checkpointer_singleton is not None:
return _async_checkpointer_singleton
try:
from deerflow.config.checkpointer_config import get_checkpointer_config
cp_cfg = get_checkpointer_config()
except Exception:
logger.exception("ai_writing: 加载 checkpointer 配置失败")
_async_checkpointer_singleton = _fallback_to_memory_saver("加载 config 抛异常")
return _async_checkpointer_singleton
if cp_cfg is None:
_async_checkpointer_singleton = _fallback_to_memory_saver(
"config.yaml 未配置 checkpointer 段"
)
return _async_checkpointer_singleton
if cp_cfg.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
_async_checkpointer_singleton = InMemorySaver()
logger.info("ai_writing: 配置 type=memory,使用 InMemorySaver")
return _async_checkpointer_singleton
if cp_cfg.type == "sqlite":
try:
import aiosqlite
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.store._sqlite_utils import (
ensure_sqlite_parent_dir,
resolve_sqlite_conn_str,
)
conn_str = resolve_sqlite_conn_str(
cp_cfg.connection_string or "ai_writing_checkpoints.db"
)
ensure_sqlite_parent_dir(conn_str)
# 直接持有 aiosqlite.Connection —— 进程生命周期,shutdown 时显式关。
conn = await aiosqlite.connect(conn_str)
saver = AsyncSqliteSaver(conn)
await saver.setup()
_async_checkpointer_singleton = saver
logger.info("ai_writing: 使用 AsyncSqliteSaver (%s)", conn_str)
return saver
except Exception as exc:
# sqlite 失败原因可能是:依赖未装 / 磁盘满 / 文件权限 / 路径不存在 /
# 文件被外部进程独占锁。任何一个都不能让 AI 写作整个不可用。
logger.exception(
"ai_writing: AsyncSqliteSaver 初始化失败,conn_str=%r",
getattr(cp_cfg, "connection_string", None),
)
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"sqlite 初始化失败: {type(exc).__name__}: {exc}"
)
return _async_checkpointer_singleton
if cp_cfg.type == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from psycopg_pool import AsyncConnectionPool
if not cp_cfg.connection_string:
raise ValueError("checkpointer.connection_string 是 postgres 必填项")
pool = AsyncConnectionPool(cp_cfg.connection_string, open=False)
await pool.open()
saver = AsyncPostgresSaver(pool)
await saver.setup()
_async_checkpointer_singleton = saver
logger.info("ai_writing: 使用 AsyncPostgresSaver")
return saver
except Exception as exc:
logger.exception("ai_writing: AsyncPostgresSaver 初始化失败")
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"postgres 初始化失败: {type(exc).__name__}: {exc}"
)
return _async_checkpointer_singleton
# 未知 type —— config 配错也不能让服务挂
_async_checkpointer_singleton = _fallback_to_memory_saver(
f"未知 checkpointer.type={cp_cfg.type!r}"
)
return _async_checkpointer_singleton
async def get_ai_writing_graph():
"""懒加载并返回 AI 写作 graph 单例。
第一次调用按 config 选定的 checkpointer 构建 graph 并缓存;后续调用直接
返回缓存。测试可在调用前通过 ``monkeypatch.setattr`` 直接替换模块级
``AI_WRITING_GRAPH`` —— 本函数会优先返回已设的值。
"""
global AI_WRITING_GRAPH
if AI_WRITING_GRAPH is not None:
return AI_WRITING_GRAPH
checkpointer = await _get_async_checkpointer()
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
return AI_WRITING_GRAPH
def get_ai_writing_checkpointer():
"""返回当前已实例化的 checkpointer(清理任务用它来删 thread)。
若尚未懒构建过则返回 None。
"""
return _async_checkpointer_singleton
async def shutdown_ai_writing_graph() -> None:
"""关闭 saver 持有的底层连接,并清空 graph 单例。Lifespan shutdown 调用。"""
global AI_WRITING_GRAPH, _async_checkpointer_singleton
saver = _async_checkpointer_singleton
_async_checkpointer_singleton = None
AI_WRITING_GRAPH = None
if saver is None:
return
# aiosqlite saver: saver.conn 是 aiosqlite.Connection
conn = getattr(saver, "conn", None)
if conn is not None and hasattr(conn, "close"):
try:
await conn.close()
except Exception:
logger.warning("ai_writing: 关闭 checkpointer connection 失败", exc_info=True)
# postgres saver: saver.pool 是 AsyncConnectionPool
pool = getattr(saver, "pool", None)
if pool is not None and hasattr(pool, "close"):
try:
await pool.close()
except Exception:
logger.warning("ai_writing: 关闭 checkpointer pool 失败", exc_info=True)
@@ -0,0 +1,242 @@
"""节点共用工具:JSON 解析容错、LLM 创建。
弱模型经常输出不规范 JSON(中文标点、结尾多余逗号、被 markdown 包裹、
夹带解释文字、嵌套对象)。parse_json 用多级降级策略尽量抢救出结构化结果。
"""
from __future__ import annotations
import json
import logging
import re
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
logger = logging.getLogger(__name__)
def get_model(config: RunnableConfig, model_name: str | None = None):
"""创建对话模型。
model_name 显式传入时优先使用(如素材收集专家的检索词/排序专用快模型),
否则回退到 configurable.model_name(用户在写作表单里选的主模型)。
"""
from deerflow.models import create_chat_model
name = model_name or config.get("configurable", {}).get("model_name", "") or None
return create_chat_model(name=name)
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
_FULLWIDTH_FIX = {
"“": '"', # 左双引号 “
"”": '"', # 右双引号 ”
"‘": "'", # 左单引号 ‘
"’": "'", # 右单引号 ’
":": ":", # 全角冒号 :
",": ",", # 全角逗号 ,
"(": "(", # 全角左括号 (
")": ")", # 全角右括号 )
"{": "{", # 全角左花括号 {
"}": "}", # 全角右花括号 }
"[": "[", # 全角左方括号 [
"]": "]", # 全角右方括号 ]
"【": "[", # 黑方括号 【
"】": "]", # 黑方括号 】
}
def _extract_balanced(text: str) -> str | None:
"""从文本中提取第一个括号平衡的 {...} 块(支持嵌套,忽略字符串内的括号)。"""
start = text.find("{")
if start == -1:
return None
depth = 0
in_str = False
escape = False
for i in range(start, len(text)):
ch = text[i]
if escape:
escape = False
continue
if ch == "\\":
escape = True
continue
if ch == '"':
in_str = not in_str
continue
if in_str:
continue
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return text[start:i + 1]
return None
def _repair_json(text: str) -> str:
"""尽力修复弱模型常见的 JSON 错误:全角标点、结尾多余逗号。"""
for bad, good in _FULLWIDTH_FIX.items():
text = text.replace(bad, good)
# 去掉对象/数组结尾多余的逗号: ,} 或 ,]
text = re.sub(r",(\s*[}\]])", r"\1", text)
return text
def _try_load(text: str) -> dict | None:
try:
result = json.loads(text)
except Exception:
return None
return result if isinstance(result, dict) else None
def parse_json(text: str, fallback: dict) -> dict:
"""从 LLM 响应中提取 JSON 对象,多级降级,失败时返回 fallback。"""
if not text or not text.strip():
return fallback
text = text.strip()
# 1. 直接解析
if (r := _try_load(text)) is not None:
return r
# 2. markdown 代码块内容
m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
if m:
fenced = m.group(1).strip()
if (r := _try_load(fenced)) is not None:
return r
text = fenced # 用代码块内容继续后续步骤
# 3. 平衡括号提取(处理夹带解释文字、嵌套对象的情况)
candidate = _extract_balanced(text)
if candidate:
if (r := _try_load(candidate)) is not None:
return r
# 4. 修复后再试
if (r := _try_load(_repair_json(candidate))) is not None:
return r
# 5. 整体修复后兜底
if (r := _try_load(_repair_json(text))) is not None:
return r
logger.warning("JSON parse failed, using fallback. raw=%r", text[:200])
return fallback
def _content_to_text(content) -> str:
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
if isinstance(content, str):
return content
if isinstance(content, list):
return "".join(
part.get("text", "") if isinstance(part, dict) else str(part)
for part in content
)
return str(content)
# 中文常见标点:参与「字数」统计,与 Word 字数统计 / 用户直觉口径对齐
_CJK_PUNCTUATION = ",。、;:?!“”‘’()《》【】「」『』〈〉—…·~"
def count_words(text: str) -> int:
"""统计中文语境下的「字数」。
口径:汉字逐字计 + 中文标点逐个计 + 连续数字/英文各算 1 字。
不计入空白、markdown 语法符号(# * - 等)与 ASCII 标点。
前后端共用同一口径(前端 utils/wordCount.ts 实现一致)。
"""
if not text:
return 0
cjk = len(re.findall(r"[一-鿿]", text))
cjk_punct = sum(1 for c in text if c in _CJK_PUNCTUATION)
alnum_runs = len(re.findall(r"[A-Za-z0-9]+", text))
return cjk + cjk_punct + alnum_runs
class ThinkTagParser:
"""流式 <think>...</think> 标签解析器。
模型可能在任意 chunk 边界截断标签,通过尾部缓冲处理跨 chunk 边界情况。
调用 feed(chunk) 返回 list[tuple[bool, str]],每项为 (is_thinking, text)。
"""
_OPEN_TAG = "<think>"
_CLOSE_TAG = "</think>"
def __init__(self) -> None:
self._in_think = False
self._buf = "" # 潜在标签的部分匹配缓冲
def feed(self, chunk: str) -> list[tuple[bool, str]]:
"""处理一个流式 chunk,返回 (is_thinking, text) 片段列表。"""
self._buf += chunk
result: list[tuple[bool, str]] = []
while self._buf:
if self._in_think:
tag = self._CLOSE_TAG
idx = self._buf.find(tag)
if idx != -1:
if idx > 0:
result.append((True, self._buf[:idx]))
self._buf = self._buf[idx + len(tag):]
self._in_think = False
else:
partial = self._partial_suffix(self._buf, tag)
if partial:
result.append((True, self._buf[:-partial]))
self._buf = self._buf[-partial:]
else:
result.append((True, self._buf))
self._buf = ""
break
else:
tag = self._OPEN_TAG
idx = self._buf.find(tag)
if idx != -1:
if idx > 0:
result.append((False, self._buf[:idx]))
self._buf = self._buf[idx + len(tag):]
self._in_think = True
else:
partial = self._partial_suffix(self._buf, tag)
if partial:
result.append((False, self._buf[:-partial]))
self._buf = self._buf[-partial:]
else:
result.append((False, self._buf))
self._buf = ""
break
return [(is_t, t) for (is_t, t) in result if t]
@staticmethod
def _partial_suffix(text: str, tag: str) -> int:
"""返回 text 尾部与 tag 前缀匹配的最长长度(>0 说明可能是部分标签)。"""
max_check = min(len(tag) - 1, len(text))
for length in range(max_check, 0, -1):
if tag.startswith(text[-length:]):
return length
return 0
async def llm_json(system: str, user: str, fallback: dict, config: RunnableConfig, model_name: str | None = None) -> dict:
"""调用 LLM,解析 JSON 输出,失败用 fallback。
model_name 可覆盖默认模型(如检索词/排序使用配置的专用快模型)。
"""
messages = [SystemMessage(content=system), HumanMessage(content=user)]
try:
# get_model 放进 try:模型名在 config.yaml 找不到时 create_chat_model 会抛
# ValueError,若留在 try 外会逐层上抛使整个图硬崩溃。
model = get_model(config, model_name=model_name)
resp = await model.ainvoke(messages)
return parse_json(_content_to_text(resp.content), fallback)
except Exception as e:
logger.error("llm_json failed: %s", e, exc_info=True)
return {**fallback, "_llm_error": str(e)}
@@ -0,0 +1,180 @@
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
from __future__ import annotations
import asyncio
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, ReviewResult, ReviewIssue
from ..prompts.editor_prompts import (
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
logger = logging.getLogger(__name__)
_DEFAULT_PASS_THRESHOLD = 75
def _materials_summary(pkg: dict) -> str:
lines = []
for m in (pkg.get("materials") or [])[:6]:
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:200]}")
return "\n".join(lines) or "(无素材)"
def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
issues = []
for item in (raw_issues or []):
issues.append(ReviewIssue(
category=category,
severity=item.get("severity", "minor"),
location=item.get("location", ""),
issue=item.get("issue", ""),
suggestion=item.get("suggestion", ""),
))
return issues
async def _stream_one_review(
model, user_prompt: str, category: str,
sse_queue: asyncio.Queue | None, session_id: str,
) -> str:
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
parser = ThinkTagParser()
async for chunk in model.astream([
SystemMessage(content=EDITOR_SYSTEM),
HumanMessage(content=user_prompt),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("review_chunk", {
"session_id": session_id,
"category": category,
"text": text,
"is_thinking": is_thinking,
}))
return full_text
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
draft = state.get("current_draft")
if not draft:
return {
"status": "error",
"progress_events": [{"type": "error", "agent_name": "编辑", "message": "草稿为空,无法审核"}],
}
article = draft["full_markdown"]
pkg = state.get("material_package") or {}
mats = _materials_summary(pkg)
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
REVIEW_TASKS = [
("fact", FACT_REVIEW_USER.format(materials_summary=mats, article=article)),
("logic", LOGIC_REVIEW_USER.format(article=article)),
("language", LANGUAGE_REVIEW_USER.format(article=article)),
]
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
except Exception as e:
logger.error("editor model creation failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error", "agent_name": "编辑",
"title": "模型错误", "detail": str(e), "message": "审核失败",
}],
}
results: dict[str, dict] = {}
review_errors: list[str] = [] # 分项审查的模型异常,供完成后非致命提示
_CATEGORY_LABEL = {"fact": "事实性", "logic": "逻辑性", "language": "语言文字"}
for category, user_prompt in REVIEW_TASKS:
fallback = {"score": 80, "issues": []}
try:
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
res = parse_json(full_text, fallback)
except Exception as e:
logger.warning("editor review [%s] failed: %s", category, e)
res = fallback
review_errors.append(f"「{_CATEGORY_LABEL.get(category, category)}审查」{type(e).__name__}: {e}")
results[category] = res
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
if sse_queue is not None:
issues = _parse_issues(res.get("issues", []), category)
await sse_queue.put(("review_item_done", {
"session_id": session_id,
"category": category,
"score": int(res.get("score", 80)),
"issues": [dict(iss) for iss in issues],
}))
else:
logger.debug("editor [%s] done (no sse_queue)", category)
# 汇总
fact_score = int(results["fact"].get("score", 80))
logic_score = int(results["logic"].get("score", 80))
lang_score = int(results["language"].get("score", 80))
overall_score = (fact_score + logic_score + lang_score) // 3
all_issues: list[ReviewIssue] = (
_parse_issues(results["fact"].get("issues", []), "fact") +
_parse_issues(results["logic"].get("issues", []), "logic") +
_parse_issues(results["language"].get("issues", []), "language")
)
all_issues.sort(key=lambda x: (0 if x["severity"] == "major" else 1))
verdict = "pass" if overall_score >= pass_threshold else "reject"
review = ReviewResult(
verdict=verdict,
fact_score=fact_score,
logic_score=logic_score,
language_score=lang_score,
overall_score=overall_score,
pass_threshold=pass_threshold,
revision_notes=all_issues,
)
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
notices: list[dict] = []
if review_errors:
notices.append({
"type": "notice",
"agent_name": "编辑",
"title": "模型错误",
"detail": "部分审查项调用失败,已按默认分兜底。原因:" + ";".join(review_errors[:3]),
})
return {
"review_result": review,
"progress_events": [
{"type": "step_start", "agent_name": "编辑", "message": "开始审核"},
*notices,
{
"type": "review_ready",
"agent_name": "编辑",
"message": f"审核完成,综合得分 {overall_score},结论:{'通过' if verdict == 'pass' else '打回修改'}",
},
],
}
@@ -0,0 +1,131 @@
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
from __future__ import annotations
import re
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState
from ._utils import llm_json
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
_DEFAULT_ARTICLE_TYPES: list[dict] = [
{"key": "news", "label": "新闻资讯"},
{"key": "analysis", "label": "分析报道"},
{"key": "feature", "label": "深度特稿"},
{"key": "report", "label": "调研报告"},
]
_SYSTEM_TEMPLATE = """你是一名写作助手。
你的任务:分析用户的写作意图,提取结构化的写作配置。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- article_type 只能是以下之一:{type_options}。
【完整示例】
输入意图:帮我写一篇面向行业专家的深度文章,分析今年新能源汽车出口为什么大涨,大概两千字
正确输出:
{{"article_type": "feature", "target_audience": "行业专家", "word_count_target": 2000, "refined_intent": "分析今年新能源汽车出口高速增长的原因"}}"""
_USER_TEMPLATE = """用户写作意图:{user_intent}
请分析并输出以下 JSON(如果意图中未明确某字段,给出合理默认值):
{{{{
"article_type": "{type_options} 之一",
"target_audience": "目标读者描述",
"word_count_target": 800,
"refined_intent": "精炼后的写作意图(一句话)"
}}}}"""
def _build_type_options(types: list[dict]) -> str:
"""将类型列表格式化为提示词中的枚举说明,如 news(新闻资讯)/ analysis(分析报道)。"""
return " / ".join(f'{t["key"]}({t["label"]})' for t in types)
def _coerce_word_count(raw) -> int | None:
"""稳健地把模型返回的字数字段转成正整数;无法解析时返回 None。
弱模型(尤其老模型被陌生词带偏时)常返回「约800字」「两千」「800.0」
这类非纯数字形式,直接 int() 会抛 ValueError/TypeError 使整个图崩溃。
此处用「提取首个数字串」策略容错,解析不出就返回 None 让调用方保留默认值。
"""
if raw is None:
return None
if isinstance(raw, bool): # bool 是 int 子类,需先排除
return None
if isinstance(raw, (int, float)):
value = int(raw)
return value if value > 0 else None
if not isinstance(raw, str):
# dict / list 等结构化类型:str() 化后可能误含数字,直接判为无法解析
return None
m = re.search(r"\d[\d,]*", raw)
if not m:
return None
try:
value = int(m.group().replace(",", ""))
except ValueError:
return None
return value if value > 0 else None
async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> dict:
# 从 configurable 读取动态类型列表,未配置时退回默认值
configurable = (config or {}).get("configurable", {})
valid_article_types: list[dict] = configurable.get("valid_article_types") or _DEFAULT_ARTICLE_TYPES
valid_keys: list[str] = [t["key"] for t in valid_article_types]
default_key: str = valid_keys[0] if valid_keys else "analysis"
type_options = _build_type_options(valid_article_types)
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
user = _USER_TEMPLATE.format(
user_intent=state["user_intent"],
type_options=" | ".join(valid_keys),
)
fallback_type = state.get("article_type", default_key)
if fallback_type not in valid_keys:
fallback_type = default_key
result = await llm_json(
system,
user,
fallback={
"article_type": fallback_type,
"target_audience": state.get("target_audience", "普通读者"),
"word_count_target": state.get("word_count_target", 800),
"refined_intent": state["user_intent"],
},
config=config,
)
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
# 流程仍可继续,但用户需知道结构化配置可能不准)。
progress_events: list[dict] = []
if result.get("_llm_error"):
progress_events.append({
"type": "notice",
"agent_name": "系统",
"title": "模型错误",
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
})
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
updates: dict = {"status": "researching", "progress_events": progress_events}
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
raw_type = result.get("article_type", "")
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
if result.get("target_audience"):
updates["target_audience"] = result["target_audience"]
word_count = _coerce_word_count(result.get("word_count_target"))
if word_count is not None:
updates["word_count_target"] = word_count
if result.get("refined_intent"):
updates["user_intent"] = result["refined_intent"]
return updates
@@ -0,0 +1,271 @@
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
from __future__ import annotations
import asyncio
import logging
import re
import uuid
from typing import List
from langchain_core.runnables import RunnableConfig
from ..search import get_search_provider, SearchResult
from ..state import AIWritingState, Material, MaterialPackage
from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
)
from ._utils import llm_json
logger = logging.getLogger(__name__)
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 20
_DEFAULT_MAX_MATERIALS = 20
_MIN_MAX_MATERIALS = 1
_MAX_MAX_MATERIALS = 100
# 素材检索关键词数量边界:用户可指定 1~10 个,非法值回退默认 6
_DEFAULT_KEYWORD_COUNT = 6
_MAX_KEYWORD_COUNT = 10
def _resolve_keyword_count(raw: object) -> int:
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
try:
n = int(raw) # type: ignore[arg-type]
except (TypeError, ValueError):
return _DEFAULT_KEYWORD_COUNT
return max(1, min(_MAX_KEYWORD_COUNT, n))
def _resolve_max_materials(raw: object) -> int:
"""把 config.yaml 配置的素材总量上限规整到 [1, 100],非法值回退默认 20。"""
try:
n = int(raw) # type: ignore[arg-type]
except (TypeError, ValueError):
return _DEFAULT_MAX_MATERIALS
return max(_MIN_MAX_MATERIALS, min(_MAX_MAX_MATERIALS, n))
# 占位/示意性关键词(模型误抄提示词模板时产生),检索前剔除
_PLACEHOLDER_KEYWORD_RE = re.compile(r"^(关键词\d*|keyword\d*|kw\d*|\.+|…+)$", re.IGNORECASE)
def _clean_keywords(raw: object, fallback: str) -> list[str]:
"""清洗 LLM 产出的关键词:去空白、丢弃占位符(如「关键词1」「...」)、去重。
全部被滤掉时回退为写作意图本身,避免拿占位词检索出一堆无关素材。
"""
cleaned: list[str] = []
for kw in raw if isinstance(raw, list) else []:
s = str(kw).strip()
if not s or _PLACEHOLDER_KEYWORD_RE.match(s):
continue
cleaned.append(s)
deduped = list(dict.fromkeys(cleaned))
return deduped or [fallback]
def _per_keyword_limit(max_materials: int, total_keywords: int) -> int:
"""把素材总量上限均分到每个检索词上的检索条数(向上取整)。
向上取整保证合并去重后素材数仍能接近总量上限;最终会再统一截断到上限。
total_keywords <= 0 时回退为总量上限本身。
"""
if total_keywords <= 0:
return max_materials
return (max_materials + total_keywords - 1) // total_keywords
def _notice(title: str, detail: str, raw: str = "") -> dict:
"""构造一个非致命告警事件,前端在时间线显示「标题 + 真实原因」,不打断写作流程。
raw 非空时附带接口的原始错误响应体,前端默认折叠展示。
"""
event = {"type": "notice", "agent_name": "素材收集专家", "title": title, "detail": detail}
if raw:
event["raw"] = raw
return event
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
intent = state["user_intent"]
article_type = state.get("article_type", "analysis")
target_audience = state.get("target_audience", "普通读者")
# 检查是否是补充搜索(PAUSE-1 re_search action)
intervention = state.get("last_user_intervention") or {}
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
# 素材不足求助点触发的「补充检索」:保留原大纲,检索完直接回写作、不走素材确认/大纲
is_supplement = intervention.get("action") == "supplement_search"
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
cfg = config.get("configurable", {})
sse_queue = cfg.get("sse_queue")
sse_session_id: str = cfg.get("sse_session_id", "")
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
rank_model: str | None = ai_writing_config.get("rank_model") or None
# 素材总量上限(config.yaml: ai_writing.max_materials),按检索词数均分到每个词
max_materials = _resolve_max_materials(ai_writing_config.get("max_materials"))
async def _emit(message: str) -> None:
if sse_queue is None:
return
await sse_queue.put(("research_progress", {
"session_id": sse_session_id,
"message": message,
}))
# Step 1: 提取关键词
await _emit("正在生成检索词…")
kw_result = await llm_json(
EXTRACT_KEYWORDS_SYSTEM,
EXTRACT_KEYWORDS_USER.format(
user_intent=intent,
article_type=article_type,
target_audience=target_audience,
keyword_count=keyword_count,
),
fallback={"keywords": [intent]},
config=config,
model_name=keyword_model,
)
# 清洗关键词:剔除占位符 / 空值,全空时回退写作意图
keywords: list[str] = _clean_keywords(kw_result.get("keywords"), intent)
if extra_keywords:
keywords = list(dict.fromkeys(keywords + extra_keywords))
# Step 2: 并发搜索
provider = get_search_provider(ai_writing_config)
# 按用户指定的关键词数量截断;补充搜索时不截断用户额外添加的关键词
search_limit = keyword_count
if extra_keywords:
search_limit = max(search_limit, len(keywords))
search_keywords = keywords[:search_limit]
total = len(search_keywords)
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
per_keyword = _per_keyword_limit(max_materials, total)
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
completed = 0
async def _search_one(kw: str) -> list[SearchResult]:
nonlocal completed
try:
return await provider.search(kw, max_results=per_keyword)
finally:
completed += 1
if completed < total:
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
else:
await _emit(f"已完成全部 {total} 次检索")
search_tasks = [_search_one(kw) for kw in search_keywords]
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
# 合并去重
seen_urls: set[str] = set()
raw_materials: list[Material] = []
existing_pkg = state.get("material_package")
if existing_pkg and intervention.get("action") != "re_search":
for m in existing_pkg.get("materials", []):
seen_urls.add(m.get("url", ""))
raw_materials.append(m)
search_errors: list[str] = []
for results in all_results_nested:
if isinstance(results, Exception):
logger.warning("search failed: %s", results)
search_errors.append(f"{type(results).__name__}: {results}")
continue
for r in results:
if r.url and r.url in seen_urls:
continue
seen_urls.add(r.url)
raw_materials.append(Material(
id=f"m{len(raw_materials)+1:03d}",
title=r.title,
content=r.content[:500],
source=r.source,
published_at=r.published_at,
relevance_score=r.relevance_score,
source_type=r.source_type,
url=r.url,
recUuid=r.recUuid,
))
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
search_errors.extend(getattr(provider, "errors", []))
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
await _emit("检索完成,正在筛选排序素材…")
rank_error = ""
if len(raw_materials) > max_materials:
materials_list = "\n".join(
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
for m in raw_materials
)
rank_result = await llm_json(
RANK_MATERIALS_SYSTEM,
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
fallback={"ranked_ids": [m["id"] for m in raw_materials[:max_materials]], "summary": f"共搜索到{len(raw_materials)}条素材"},
config=config,
model_name=rank_model,
)
rank_error = rank_result.get("_llm_error", "")
ranked_ids: list[str] = rank_result.get("ranked_ids", [m["id"] for m in raw_materials])
id_to_mat = {m["id"]: m for m in raw_materials}
sorted_materials = [id_to_mat[mid] for mid in ranked_ids if mid in id_to_mat]
# 加入未被排序的素材
sorted_ids_set = set(ranked_ids)
for m in raw_materials:
if m["id"] not in sorted_ids_set:
sorted_materials.append(m)
materials = sorted_materials[:max_materials]
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
else:
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:max_materials]
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
if not materials:
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
notices: list[dict] = []
if kw_result.get("_llm_error"):
notices.append(_notice("模型错误", f"检索词生成失败,已用默认关键词兜底。原因:{kw_result['_llm_error']}"))
if rank_error:
notices.append(_notice("模型错误", f"素材排序失败,已按相关度兜底排序。原因:{rank_error}"))
if search_errors:
uniq = list(dict.fromkeys(search_errors))
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
notices.append(_notice(
"ES 波动导致检索异常",
";".join(uniq[:3]),
raw="\n\n----------\n\n".join(raw_dumps[:3]),
))
return {
"material_package": pkg,
# 补充检索:直接回写作(沿用原大纲);常规检索:进入素材确认暂停点
"status": "writing_draft" if is_supplement else "awaiting_material_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
*notices,
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
],
}
@@ -0,0 +1,318 @@
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import logging
import re
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, DraftArticle
from ..prompts.writer_prompts import (
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
)
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
logger = logging.getLogger(__name__)
# 章节正文里的 Markdown 标题行(允许前导空白)
_HEADING_PREFIX_RE = re.compile(r"^[ \t]*#{1,6}[ \t]+", re.MULTILINE)
def _normalize_section_headings(text: str) -> str:
"""把章节正文里所有 Markdown 标题行统一压成三级标题(###)。
作家被要求只用 ### 作为「要点」小标题;模型偶尔误用 #/##/#### 时在此纠正,
保证文档标题层级符合「文章 # → 章节 ## → 要点 ###」,
前端目录栏据此才能把要点正确识别为二级标题。
"""
return _HEADING_PREFIX_RE.sub("### ", text)
# 疑似「拒绝 / 元话语」的强特征短语:模型把对用户说的话当成正文输出时出现,
# 这些短语几乎不可能出现在正常文章里,用作安全网识别。
_REFUSAL_MARKERS = (
"我无法完成", "无法完成此任务", "无法完成该任务", "无法完成这个任务",
"无法完成本次", "无法写出符合", "无法撰写这篇", "无法基于现有素材",
"无法基于当前素材", "无法基于提供的素材",
)
def _looks_like_refusal(text: str) -> bool:
"""判断一段章节产出是否疑似「拒绝 / 元话语」而非真正的文章正文。
模型遇到素材与意图不匹配时,可能把「我无法完成此任务」之类对用户说的话
当成正文输出。这里用强特征短语做启发式识别(安全网,非强校验)。
"""
if not text or not text.strip():
return False
return any(marker in text for marker in _REFUSAL_MARKERS)
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
lines = []
for mid in material_ids:
if mid in mat_map:
m = mat_map[mid]
lines.append(f"[{mid}] {m['title']}: {m['content'][:300]}")
if not lines:
for m in pkg.get("materials", [])[:2]:
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:300]}")
return "\n".join(lines) if lines else "(无相关素材)"
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
outline = state.get("current_outline")
if not outline:
return {
"status": "error",
"progress_events": [{"type": "error", "agent_name": "作家", "message": "大纲为空,无法写作"}],
}
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
revision_count = state.get("revision_count", 0)
# 写作模式;用户在「素材不足求助」里选「按通用知识续写」的章节强制走宽松
writing_mode = state.get("writing_mode", "loose")
loose_titles: set[str] = set(intervention.get("_loose_sections") or [])
# SSE 流式推送所需
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
# 修改意见(来自用户或编辑)
revision_notes = ""
if intervention.get("action") == "user_revise":
notes = intervention.get("_user_notes") or intervention.get("user_revision_notes", "")
if notes:
revision_notes = f"\n用户修改意见:{notes}"
elif intervention.get("action") in ("accept_review", "custom_revise"):
review = state.get("review_result") or {}
issues = review.get("revision_notes", [])
extra = intervention.get("_extra_notes") or intervention.get("user_revision_notes", "")
parts = []
for iss in issues[:5]:
parts.append(f"- [{iss.get('severity','minor')}] {iss.get('location','')}: {iss.get('suggestion','')}")
if extra:
parts.append(f"- 用户补充:{extra}")
if parts:
revision_notes = "\n修改要求:\n" + "\n".join(parts)
word_count_target = state.get("word_count_target", 800)
sections = outline.get("sections", [])
n_sections = max(len(sections), 1)
# 单节硬下限:总目标 / 节数,乘 1.1 过量系数补偿模型对短目标的天然欠量;单节至少 150 字
words_per_section = max(150, int(word_count_target * 1.1 / n_sections))
article_title = outline.get("title", "文章")
# 兼容后端 snake_case 与前端编辑后回传的 camelCase
def _section_title(idx: int, sec: dict) -> str:
return sec.get("section_title") or sec.get("sectionTitle") or f"章节{idx + 1}"
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
try:
model = get_model(config)
except Exception as e:
logger.error("draft model creation failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error", "agent_name": "作家",
"title": "模型错误", "detail": str(e), "message": "草稿生成失败",
}],
}
section_titles: list[str] = []
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
blocked_sections: list[dict] = [] # 严格模式下素材不足、无法写作的章节
for i, section in enumerate(sections):
sec_title = _section_title(i, section)
key_points = section.get("key_points") or section.get("keyPoints") or []
mat_ids = section.get("material_ids") or section.get("materialIds") or []
section_materials = _get_section_materials(mat_ids, pkg)
key_points_str = "\n".join(f"- {p}" for p in key_points)
# 文章整体结构:让作家看到全文章节与自己的定位,避免章节间内容重复、引言结论脱节
outline_context = "\n".join(
f"{j + 1}. {_section_title(j, s)}" + (" ← 当前正在写这一章" if j == i else "")
for j, s in enumerate(sections)
)
# 本章节是否走严格模式(全局严格 且 未被用户标记为「通用知识续写」)
section_is_strict = writing_mode == "strict" and sec_title not in loose_titles
mode_section = WRITER_MODE_STRICT if section_is_strict else WRITER_MODE_LOOSE
user_msg = DRAFT_SECTION_USER.format(
article_title=article_title,
outline_context=outline_context,
section_title=sec_title,
key_points=key_points_str,
section_materials=section_materials,
revision_section=revision_notes,
mode_section=mode_section,
target_words=words_per_section,
)
section_buf = "" # 只累积正文(非思考)用于拼接 Markdown
parser = ThinkTagParser()
try:
async for chunk in model.astream([
SystemMessage(content=DRAFT_SECTION_SYSTEM),
HumanMessage(content=user_msg),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
section_buf += text
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": text,
"is_thinking": is_thinking,
"is_section_done": False,
}))
# 章节完成
if sse_queue is not None:
await sse_queue.put(("draft_chunk", {
"session_id": session_id,
"section_index": i,
"section_title": sec_title,
"text": "",
"is_section_done": True,
}))
cleaned_buf = section_buf.strip()
blocked_reason = ""
# 严格模式:作家用哨兵声明素材不足,或仍输出了拒绝话术 → 标记为待求助章节
if section_is_strict and SECTION_BLOCKED_MARKER in cleaned_buf:
blocked_reason = (
cleaned_buf.split(SECTION_BLOCKED_MARKER, 1)[1].strip()[:120]
or "素材不足以支撑本章节写作"
)
elif _looks_like_refusal(cleaned_buf):
if section_is_strict:
blocked_reason = "素材不足以支撑本章节写作"
else:
# 宽松模式:模型仍拒绝,安全网占位,不打断流程
logger.warning("draft section [%s] looks like a refusal, replaced with placeholder", sec_title)
section_errors.append(f"「{sec_title}」生成内容疑似拒绝/无关,已占位")
cleaned_buf = "(本节内容生成异常,建议重新生成或补充相关素材)"
if blocked_reason:
blocked_sections.append({"section_title": sec_title, "reason": blocked_reason})
cleaned_buf = f"(本节素材不足:{blocked_reason})"
section_titles.append(sec_title)
section_bufs.append(cleaned_buf)
except Exception as e:
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
section_titles.append(sec_title)
section_bufs.append("(内容生成失败,请重试)")
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
# 严格模式下有章节无法基于素材写作 → 暂停求助用户,不产出半成品草稿
if blocked_sections:
return {
"blocked_sections": blocked_sections,
"status": "awaiting_section_help",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
{
"type": "notice", "agent_name": "作家", "title": "素材不足",
"detail": f"有 {len(blocked_sections)} 个章节缺少足够素材支撑写作,需要你决定如何处理。",
},
],
}
def _assemble() -> str:
# 章节正文里的标题统一归一到 ###(要点小标题),章节标题用 ##,文章标题用 #
body = "\n\n".join(
f"## {t}\n\n{_normalize_section_headings(b)}"
for t, b in zip(section_titles, section_bufs)
)
return f"# {article_title}\n\n{body}"
full_markdown = _assemble()
word_count = count_words(full_markdown)
# 扩写兜底:总字数不足目标 90% 时,重写并扩充最短的章节(单轮,非流式)
if section_bufs and word_count < word_count_target * 0.9:
idx = min(range(len(section_bufs)), key=lambda k: count_words(section_bufs[k]))
target_sec = sections[idx]
expand_target = count_words(section_bufs[idx]) + (word_count_target - word_count)
try:
expand_msg = DRAFT_EXPAND_USER.format(
article_title=article_title,
section_title=section_titles[idx],
key_points="\n".join(
f"- {p}" for p in (target_sec.get("key_points") or target_sec.get("keyPoints") or [])
),
section_materials=_get_section_materials(
target_sec.get("material_ids") or target_sec.get("materialIds") or [], pkg
),
existing_text=section_bufs[idx],
target_words=expand_target,
)
resp = await model.ainvoke([
SystemMessage(content=DRAFT_SECTION_SYSTEM),
HumanMessage(content=expand_msg),
])
parser = ThinkTagParser()
expanded = "".join(
t for is_thinking, t in parser.feed(_content_to_text(resp.content)) if not is_thinking
).strip()
# 仅在扩写确实更长时才采用,避免越改越短
if count_words(expanded) > count_words(section_bufs[idx]):
section_bufs[idx] = expanded
full_markdown = _assemble()
word_count = count_words(full_markdown)
except Exception as e:
logger.warning("draft expand failed for section [%s]: %s", section_titles[idx], e)
draft = DraftArticle(
title=outline.get("title", "文章"),
outline=outline,
full_markdown=full_markdown,
word_count=word_count,
revision_count=revision_count,
)
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
notices: list[dict] = []
if section_errors:
notices.append({
"type": "notice",
"agent_name": "作家",
"title": "模型错误",
"detail": "部分章节生成失败,对应段落已占位。原因:" + ";".join(section_errors[:3]),
})
return {
"current_draft": draft,
"drafts": [draft],
"status": "awaiting_draft_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
*notices,
{
"type": "draft_ready",
"agent_name": "作家",
"message": f"草稿完成,共 {word_count} 字",
"draft_title": outline.get("title", "文章"),
"draft_markdown": full_markdown,
"revision_count": revision_count,
},
],
}
@@ -0,0 +1,147 @@
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
from __future__ import annotations
import asyncio
import json
import logging
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..state import AIWritingState, Outline, OutlineSection
from ..prompts.writer_prompts import (
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
)
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
logger = logging.getLogger(__name__)
# 上下文压缩:传给大纲规划的素材摘要做条数和单条长度双重截断,防止 token 溢出。
_MAX_OUTLINE_MATERIALS = 12
_MATERIAL_SNIPPET_CHARS = 150
def _materials_summary(pkg: dict) -> str:
lines = []
for m in pkg.get("materials", [])[:_MAX_OUTLINE_MATERIALS]:
lines.append(f"[{m['id']}] {m['title']} | {m['source']} | {m['content'][:_MATERIAL_SNIPPET_CHARS]}")
return "\n".join(lines) or "(无素材)"
def _parse_outline(raw: dict) -> Outline:
sections = []
for s in raw.get("sections", []):
sections.append(OutlineSection(
section_title=s.get("section_title", ""),
key_points=s.get("key_points", []),
material_ids=s.get("material_ids", []),
))
if not sections:
sections = [OutlineSection(section_title="正文", key_points=["主要内容"], material_ids=[])]
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
intervention = state.get("last_user_intervention") or {}
feedback = ""
if intervention.get("action") == "re_outline":
feedback = intervention.get("_feedback") or intervention.get("outline_feedback") or ""
elif intervention.get("outline_feedback"):
feedback = intervention.get("outline_feedback") or ""
# 重新规划且已有大纲 → 走「编辑模式」:把当前大纲喂给模型,按用户要求调整,
# 而非从零重生成(否则「加一节/删一节」这类指令没有可编辑的基准)。
current_outline = state.get("current_outline")
is_revise = bool(feedback) and bool(current_outline)
if is_revise:
system_prompt = OUTLINE_REVISE_SYSTEM
user_prompt = OUTLINE_REVISE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
current_outline=json.dumps(current_outline, ensure_ascii=False, indent=2),
feedback=feedback,
materials_summary=_materials_summary(pkg),
)
else:
system_prompt = OUTLINE_SYSTEM
feedback_section = f"\n重新规划要求:{feedback}" if feedback else ""
user_prompt = OUTLINE_USER.format(
user_intent=state["user_intent"],
article_type=state.get("article_type", "analysis"),
target_audience=state.get("target_audience", "普通读者"),
word_count_target=state.get("word_count_target", 800),
materials_summary=_materials_summary(pkg),
feedback_section=feedback_section,
)
# SSE 流式推送
cfg = config.get("configurable", {})
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
session_id: str = cfg.get("sse_session_id", "")
full_text = "" # 只累积正文(非思考)用于 JSON 解析
parser = ThinkTagParser()
try:
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
model = get_model(config)
async for chunk in model.astream([
SystemMessage(content=system_prompt),
HumanMessage(content=user_prompt),
]):
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
if sse_queue is not None:
await sse_queue.put(("outline_chunk", {
"session_id": session_id,
"text": text,
"is_thinking": is_thinking,
}))
except Exception as e:
logger.error("outline streaming failed: %s", e, exc_info=True)
return {
"status": "error",
"progress_events": [{
"type": "error",
"agent_name": "作家",
"title": "模型错误",
"detail": str(e),
"message": "大纲生成失败",
}],
}
# 编辑模式解析失败时回退为「当前大纲」,避免把用户已有大纲弄丢
if is_revise:
fallback = current_outline
else:
fallback = {
"title": "未命名文章",
"sections": [
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
],
}
result = parse_json(full_text, fallback)
outline = _parse_outline(result)
section_count = len(outline["sections"])
return {
"current_outline": outline,
"status": "awaiting_outline_confirm",
"progress_events": [
{"type": "step_start", "agent_name": "作家", "message": "开始规划大纲"},
{"type": "outline_ready", "agent_name": "作家", "message": f"大纲已生成,共 {section_count} 个章节"},
],
}
@@ -0,0 +1,52 @@
EDITOR_SYSTEM = """你是一名资深主编(编辑),标准极高,要求严格。
你的目标:审查文章质量,给出客观评分和具体修改意见。
你的工作背景:你在一家权威媒体担任主编十五年,见过各种质量的文章。
你只能做以下事情:
1. 从指定维度审查文章
2. 给出该维度的评分(0-100 的整数)和具体问题列表
你不能改写文章,只做当前指定维度的审查。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- score 是 0-100 的整数。
- 每个问题的 severity 必须是 "major"(必须修改)或 "minor"(建议修改)。
- 文章在该维度没有问题时,issues 填空数组 []。
【完整示例】
正确输出:
{
"score": 82,
"issues": [
{"severity": "major", "location": "第二段", "issue": "引用的增长数据没有素材支撑", "suggestion": "补充素材 m002 中的具体数据,并注明来源"},
{"severity": "minor", "location": "结尾段", "issue": "结论略显仓促", "suggestion": "增加一句对未来行业趋势的判断"}
]
}"""
FACT_REVIEW_USER = """请审查以下文章的【事实准确性】维度。
判断标准:引用的数据/事实是否有素材支撑,是否有捏造或夸大。
参考素材:
{materials_summary}
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
LOGIC_REVIEW_USER = """请审查以下文章的【逻辑结构】维度。
判断标准:论点是否清晰,论据是否充分,段落是否连贯,结构是否合理。
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
LANGUAGE_REVIEW_USER = """请审查以下文章的【语言文字】维度。
判断标准:语言是否流畅,用词是否准确,有无错别字或病句。
待审文章:
{article}
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
@@ -0,0 +1,70 @@
# 注:两个 USER 提示词末尾的 `/no_think` 是 Qwen3 系列的"软开关",
# 收到当轮即跳过思考过程;对其它大模型只是一行普通文本、无副作用。
# 中文「严禁思考」强令块对所有模型生效,保留不动。
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:为写作任务提取搜索关键词。
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
你只能做以下事情:
1. 从写作意图中按要求的数量提取搜索关键词
你不能写文章,不能评价内容好坏,只做关键词提取。
【严禁思考】
- 这是一个简单的结构化抽取任务,禁止任何思考、分析、推理或规划过程。
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 ",不要用中文引号 “”。
- 关键词要具体、可搜索,避免过于宽泛的词。
【完整示例】
输入意图:写一篇分析今年新能源汽车出口增长的文章
正确输出:
{"keywords": ["新能源汽车出口", "汽车出口数据", "新能源汽车海外市场", "电动车出口增长", "汽车出口政策"]}"""
EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
请提取 {keyword_count} 个搜索关键词,用于搜集相关素材。
关键词数量必须为 {keyword_count} 个,不多不少。
关键词必须是依据上述写作意图提炼的真实检索词,不要照抄下方 JSON 模板里的占位符。
只输出 JSON(下方 ... 是占位示意,不是真实关键词):
{{"keywords": ["...", "..."]}}
/no_think"""
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
你的目标:对搜索结果按相关度排序和筛选。
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
你只能做以下事情:
1. 从提供的素材列表中,选出与写作意图最相关的素材ID,按相关度从高到低排序
你不能写文章,只做素材筛选排序。
【严禁思考】
- 这是一个简单的筛选排序任务,禁止任何思考、分析、推理或规划过程。
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- ranked_ids 中的 ID 必须来自素材列表,不要编造 ID。
【完整示例】
正确输出:
{"ranked_ids": ["m003", "m001", "m005"], "summary": "共搜索到 8 条素材,精选 5 条最相关的"}"""
RANK_MATERIALS_USER = """写作意图:{user_intent}
素材列表(每条含ID和摘要):
{materials_list}
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}
/no_think"""
@@ -0,0 +1,166 @@
OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据素材和意图,规划清晰的文章大纲。
你的工作背景:你有十年内容创作经验,擅长将零散素材组织成有逻辑的文章结构。
你只能做以下事情:
1. 分析素材,提取核心观点
2. 规划文章章节结构和每节要点
你不能写正文,只写大纲。大纲须紧扣写作意图,并尽量覆盖可用素材中的核心信息。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
- 禁止输出任何对用户说的话或对任务的评价(如"素材不足""无法规划大纲");无论素材是否充分,都必须直接产出大纲 JSON。
- 收到输入后立即、直接输出最终 JSON,不得有任何前置内容。
【要点(key_points)的特殊要求】
- 每个要点都会作为文章里的一个二级小标题呈现,所以要点必须写成"标题式短语"。
- 用简洁的名词性短语,控制在 15 字以内,不要写成完整句子,不要带句末标点。
- 每个章节规划 2-4 个要点,不要过多,避免文章被切得过碎。
【输出要求】
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
- 必须使用英文双引号 ",不要用中文引号 “”。
- 规划 3-5 个章节;material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
【完整示例】
正确输出:
{
"title": "新能源汽车出口为何高速增长",
"sections": [
{"section_title": "引言:出口数据创新高", "key_points": ["出口量同比增幅", "全球市场排名变化"], "material_ids": ["m001"]},
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
{"section_title": "结论与展望", "key_points": ["未来增长挑战", "行业趋势判断"], "material_ids": ["m004"]}
]
}"""
OUTLINE_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
目标字数:{word_count_target}字{feedback_section}
可用素材摘要:
{materials_summary}
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
# ── 重新规划(基于现有大纲的编辑)─────────────────────────────────────────────
OUTLINE_REVISE_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据用户的修改要求,调整一份【已有的文章大纲】。
你的工作背景:你有十年内容创作经验,擅长精准理解修改意图并落实到大纲结构上。
你只能做以下事情:
1. 在「当前大纲」基础上,按「用户修改要求」做调整
你不能写正文,只输出调整后的大纲。
【修改原则】(最高优先级)
- 准确识别并落实用户意图,常见操作:新增章节、删除章节、合并章节、拆分章节、调整章节顺序、修改章节标题、调整章节要点。
- 只改用户明确提到的部分:用户没要求改动的章节,必须原样保留(标题、要点、material_ids 全部不变)。
- 新增章节时:紧扣写作意图补充合理章节,并尽量关联可用素材的 material_ids。
- 章节数量完全由用户要求决定,没有上下限——用户要加就加、要减就减。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
- 禁止输出任何对用户说的话;收到输入后立即、直接输出最终 JSON。
【要点(key_points)的特殊要求】
- 每个要点会作为文章里的二级小标题呈现,写成简洁名词性短语,15 字以内,不带句末标点。
- 每个章节 2-4 个要点。
【输出要求】
- 只输出一个调整后的【完整】大纲 JSON 对象,不要解释文字,不要 markdown 代码块。
- 必须使用英文双引号 "。
- material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
- 结构与「当前大纲」一致:
{"title": "...", "sections": [{"section_title": "...", "key_points": ["..."], "material_ids": ["..."]}]}"""
OUTLINE_REVISE_USER = """写作意图:{user_intent}
文章类型:{article_type}
目标读者:{target_audience}
目标字数:{word_count_target}字
当前大纲(JSON):
{current_outline}
用户修改要求:{feedback}
可用素材摘要:
{materials_summary}
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
你只能做以下事情:
1. 根据给定的章节要点和素材写这一段内容
2. 优先采用素材中的事实与数据;素材不足时可结合你的通用知识与常识补充论述,但须紧扣写作意图与章节主题,不要脱离主题杜撰具体的数字、人名、机构名与日期
你不能修改大纲,不能写其他章节,只写当前这一段。
【严禁思考与表态】(最高优先级)
- 禁止输出任何思考、分析、规划过程,禁止"让我想想""首先""我需要""根据写作要求"之类的过渡语。
- 禁止输出 <think></think> 标签及其中的任何内容。
- 禁止输出任何对用户说的话或对任务本身的评价,例如"我无法完成此任务""您提供的素材无法支撑写作""建议您提供……素材""素材与意图不匹配"等——这类话绝不允许出现在文章里。
- 无论素材是否充分,你都必须直接产出文章正文本身;收到输入后立即开始写正文(含 ### 小标题),不要有任何前置说明。
【结构要求】
- 把本章节的每一个「章节要点」写成一个二级小标题,用 Markdown 三级标题语法:### 要点
- 每个 ### 小标题下面紧跟该要点的论述正文,正文连贯成段。
- 即使章节只有一个要点,也要为它写出对应的 ### 小标题。
- 小标题文字简洁,直接沿用给定的要点措辞,不要加序号、不要加标点。
- 只能使用 ### 这一级标题;绝不要输出 # 或 ##(文章标题、章节标题由系统统一添加)。
【输出要求】
- 直接返回这一段的内容(含 ### 小标题与正文),不要 JSON,不要解释。
- 不要写"以下是正文""本段内容如下"之类的开场白。
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
- 充分展开每个要点:每个要点至少用 2-3 句话论述,尽量结合素材中的事实与数据说理,写满要求的篇幅。"""
# ── 写作模式说明:注入 DRAFT_SECTION_USER 的 {mode_section} ───────────────────
# 章节无法基于素材写作时,作家在严格模式下输出的哨兵标记
SECTION_BLOCKED_MARKER = "[[SECTION_BLOCKED]]"
WRITER_MODE_LOOSE = """
【写作模式:宽松】
素材充分时优先采用素材;素材不足时可结合你的通用知识与常识补充论述,
但必须紧扣写作意图与本章节主题,不要脱离主题杜撰具体数字、人名、机构名。"""
WRITER_MODE_STRICT = """
【写作模式:严格依据素材】
本章节论述必须严格基于上方「相关素材」,不得使用素材之外的知识,不得杜撰任何事实与数据。
如果上方素材确实不足以支撑本章节的写作,不要勉强拼凑、不要写无关内容,
此时只输出一行:{marker} 一句话说明缺什么素材(例:{marker} 缺少关于XX的具体数据与案例),
此外不要输出任何内容。只要素材能支撑哪怕部分要点,就正常写作,不要输出该标记。""".format(
marker=SECTION_BLOCKED_MARKER
)
DRAFT_SECTION_USER = """文章标题:{article_title}
文章整体结构:
{outline_context}
当前章节:{section_title}
章节要点:{key_points}
相关素材:
{section_materials}
{revision_section}{mode_section}
请写出「{section_title}」这一章节的内容,不少于 {target_words} 字。
把上述每个章节要点写成一个 ### 二级小标题,并在标题下展开论述,论述充分、内容充实。"""
DRAFT_EXPAND_USER = """文章标题:{article_title}
当前章节:{section_title}
章节要点:{key_points}
相关素材:
{section_materials}
这一章节已有内容如下:
{existing_text}
以上内容篇幅不足。请在保留原意、事实和 ### 二级小标题结构的基础上重写并扩充这一章节,使内容不少于 {target_words} 字。
要求:
- 补充更多来自素材的事实、数据与分析,论述更充分、更具体。
- 保留并沿用原有的 ### 小标题(每个章节要点对应一个),只用 ### 这一级标题,不要输出 # 或 ##。
- 直接返回扩充后的完整章节内容(含 ### 小标题),不要解释。"""
@@ -0,0 +1,64 @@
from .base import BaseSearchProvider, SearchResult
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
"""
config.yaml 中配置:
ai_writing:
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
intranet_knowledge_base: samples
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
"""
cfg = ai_writing_config or {}
provider = cfg.get("search_provider", "internet")
if provider == "intranet":
return IntranetSearchProvider(
base_url=cfg.get("intranet_search_url", ""),
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
)
if provider == "tavily":
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
if provider == "demo":
return DemoSearchProvider()
# 默认 internet 模式:DuckDuckGo + demo fallback
region = cfg.get("ddg_region", "cn-zh")
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
class _FallbackSearchProvider(BaseSearchProvider):
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
self._primary = primary
self._fallback = fallback
@property
def errors(self) -> list[str]:
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
async def search(self, query: str, max_results: int = 5, **kwargs):
results = await self._primary.search(query, max_results=max_results, **kwargs)
if not results:
results = await self._fallback.search(query, max_results=max_results, **kwargs)
return results
async def health_check(self) -> bool:
return await self._primary.health_check() or await self._fallback.health_check()
__all__ = [
"BaseSearchProvider", "SearchResult",
"InternetSearchProvider", "IntranetSearchProvider",
"DemoSearchProvider", "DuckDuckGoSearchProvider",
"get_search_provider",
]
@@ -0,0 +1,40 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List
@dataclass
class SearchResult:
"""统一搜索结果格式 —— 内外网共用"""
title: str
content: str
source: str
published_at: str
url: str = ""
relevance_score: float = 0.0
source_type: str = "internet"
recUuid: str = ""
extra_metadata: dict = field(default_factory=dict)
class BaseSearchProvider(ABC):
"""搜索提供者抽象基类。
对接内网步骤:
1. 继承此类,创建 IntranetSearchProvider
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
4. 无需修改任何 Agent 代码
"""
@abstractmethod
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]: ...
@abstractmethod
async def health_check(self) -> bool: ...
@@ -0,0 +1,183 @@
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
from __future__ import annotations
import hashlib
from typing import List
from .base import BaseSearchProvider, SearchResult
# 演示素材库:按话题分类,用于关键词模糊匹配
_DEMO_CORPUS = [
{
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
"items": [
{
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
"source": "中国汽车工业协会",
"published_at": "2025-10-15",
"relevance_score": 0.95,
},
{
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
"source": "电池中国网",
"published_at": "2025-09-20",
"relevance_score": 0.88,
},
{
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
"source": "中国充电联盟",
"published_at": "2025-08-30",
"relevance_score": 0.82,
},
],
},
{
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
"items": [
{
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
"source": "中国信通院",
"published_at": "2025-10-08",
"relevance_score": 0.93,
},
{
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
"source": "麦肯锡中国",
"published_at": "2025-09-12",
"relevance_score": 0.87,
},
{
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
"source": "艾瑞咨询",
"published_at": "2025-10-01",
"relevance_score": 0.79,
},
],
},
{
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
"items": [
{
"title": "光伏装机量突破900GW,度电成本降至0.18元",
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
"source": "国家能源局",
"published_at": "2025-10-12",
"relevance_score": 0.92,
},
{
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
"source": "上海环境能源交易所",
"published_at": "2025-09-28",
"relevance_score": 0.85,
},
],
},
{
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
"items": [
{
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
"source": "医学界",
"published_at": "2025-09-18",
"relevance_score": 0.91,
},
],
},
{
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
"items": [
{
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
"source": "国家统计局",
"published_at": "2025-10-18",
"relevance_score": 0.90,
},
{
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
"source": "商务部",
"published_at": "2025-10-05",
"relevance_score": 0.84,
},
],
},
]
# 通用兜底素材(与任意查询都能凑合搭配)
_FALLBACK_ITEMS = [
{
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
"source": "产业研究院",
"published_at": "2025-08-15",
"relevance_score": 0.55,
},
{
"title": "政策解读:2025年产业发展专项支持资金管理办法",
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
"source": "工信部",
"published_at": "2025-07-20",
"relevance_score": 0.50,
},
{
"title": "市场竞争格局分析:头部集中与长尾分散并存",
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
"source": "普华永道",
"published_at": "2025-09-01",
"relevance_score": 0.48,
},
]
def _score_topic(query: str, keywords: list[str]) -> int:
"""简单关键词匹配打分"""
q = query.lower()
return sum(1 for kw in keywords if kw in q)
class DemoSearchProvider(BaseSearchProvider):
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
仅用于开发测试;生产环境应配置 intranet 提供者。
"""
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
items = []
if best_score > 0 and best_topic:
items.extend(best_topic["items"])
# 用通用素材补齐
for fb in _FALLBACK_ITEMS:
if len(items) >= max_results:
break
items.append(fb)
results = []
for i, it in enumerate(items[:max_results]):
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
results.append(SearchResult(
title=it["title"],
content=it["content"],
source=it["source"],
published_at=it["published_at"],
url=f"https://demo.internal/{url_hash}",
relevance_score=it["relevance_score"],
source_type="demo",
recUuid=f"demo-{i:03d}",
))
return results
async def health_check(self) -> bool:
return True
@@ -0,0 +1,81 @@
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
from __future__ import annotations
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class DuckDuckGoSearchProvider(BaseSearchProvider):
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
def __init__(self, region: str = "cn-zh", max_timeout: int = 30):
self.region = region
self.max_timeout = max_timeout
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
self.errors: list[str] = []
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(
self._sync_search, query, max_results
)
return results
except Exception as e:
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs not installed. Run: uv add ddgs")
self.errors.append("ImportError: ddgs 未安装")
return []
ddgs = DDGS(timeout=self.max_timeout)
try:
raw = ddgs.text(
query,
region=self.region,
safesearch="moderate",
max_results=max_results,
)
items = list(raw) if raw else []
except Exception as e:
logger.warning("DDGS.text failed: %s", e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
results = []
for item in items:
url = item.get("href", item.get("link", ""))
source = url.split("/")[2] if url and "/" in url else "web"
results.append(SearchResult(
title=item.get("title", ""),
content=item.get("body", item.get("snippet", ""))[:500],
source=source,
published_at=item.get("published", ""),
url=url,
relevance_score=0.75,
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return len(results) > 0
except Exception:
return False
@@ -0,0 +1,66 @@
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class InternetSearchProvider(BaseSearchProvider):
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(self._sync_search, query, max_results)
return results
except Exception as e:
logger.warning("internet search failed for query=%r: %s", query, e)
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
from deerflow.config import get_app_config
try:
from tavily import TavilyClient
except ImportError:
logger.warning("tavily not installed, falling back to demo search")
return []
config = get_app_config().get_tool_config("web_search")
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
if not api_key:
logger.warning("Tavily API key not configured, falling back to demo search")
return []
client = TavilyClient(api_key=api_key)
res = client.search(query, max_results=max_results)
results = []
for item in res.get("results", []):
content = item.get("content", "")
results.append(SearchResult(
title=item.get("title", ""),
content=content[:500],
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
published_at=item.get("published_date", ""),
url=item.get("url", ""),
relevance_score=item.get("score", 0.0),
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,114 @@
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
from __future__ import annotations
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
_DEFAULT_KB = "samples"
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
_RAW_SNIPPET_LIMIT = 2000
class IntranetSearchProvider(BaseSearchProvider):
"""内网知识库搜索接口适配器。
接口:POST {endpoint}
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
"""
def __init__(
self,
base_url: str = "",
knowledge_base_name: str = _DEFAULT_KB,
timeout: int = 120,
**_ignored,
):
self.endpoint = base_url or _DEFAULT_ENDPOINT
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
# 内网 ES 接口可能较慢,超时放宽到 2 分钟
self.timeout = timeout
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
self.errors: list[str] = []
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
self.raw_responses: list[str] = []
def _record_raw(self, body: str) -> None:
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
body = (body or "").strip()
if not body:
return
if len(body) > _RAW_SNIPPET_LIMIT:
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
self.raw_responses.append(body)
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
import httpx
payload = {
"query": query,
"knowledge_base_name": self.knowledge_base_name,
"ts": "yes",
}
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
raw_text = ""
try:
async with httpx.AsyncClient(timeout=self.timeout) as client:
resp = await client.post(self.endpoint, json=payload)
resp.raise_for_status()
raw_text = resp.text
data = resp.json()
except httpx.HTTPStatusError as e:
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
self._record_raw(e.response.text if e.response is not None else "")
return []
except Exception as e:
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
self._record_raw(raw_text)
return []
# 正常情况下接口直接返回数组;兼容个别情况下包了一层 data/results 的结构
if isinstance(data, list):
items = data
elif isinstance(data, dict):
items = data.get("data") or data.get("results") or []
else:
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
self._record_raw(raw_text)
return []
results: List[SearchResult] = []
for i, item in enumerate(items[:max_results]):
md = (item or {}).get("metadata") or {}
content = md.get("content") or (item or {}).get("page_content") or ""
results.append(SearchResult(
title=str(md.get("m_title") or "").strip() or "(无标题)",
content=str(content)[:500],
source=str(md.get("source1") or "").strip() or "内网知识库",
published_at=str(md.get("m_publish") or "").strip(),
url="", # 内网素材的原文地址由前端依据 recUuid 拼接
# 接口未返回相关度分值,按 ES 返回顺序赋递减分以保留排序
relevance_score=max(0.5, 0.95 - i * 0.03),
source_type="intranet",
recUuid=str(md.get("recUuid") or "").strip(),
))
return results
async def health_check(self) -> bool:
try:
await self.search("测试", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,121 @@
from typing import Annotated, List, Literal, Optional, TypedDict
import operator
# ── 素材 ──────────────────────────────────────────────────────────────────────
class Material(TypedDict):
id: str
title: str
content: str # 截断到 500 字
source: str
published_at: str
relevance_score: float
source_type: str # "internet" | "intranet"
url: str
recUuid: str
class MaterialPackage(TypedDict):
keywords: List[str]
materials: List[Material]
summary: str
# ── 大纲 ──────────────────────────────────────────────────────────────────────
class OutlineSection(TypedDict):
section_title: str
key_points: List[str]
material_ids: List[str]
class Outline(TypedDict):
title: str
sections: List[OutlineSection]
# ── 草稿 ──────────────────────────────────────────────────────────────────────
class DraftArticle(TypedDict):
title: str
outline: Outline
full_markdown: str
word_count: int
revision_count: int
# ── 编辑审核 ──────────────────────────────────────────────────────────────────
class ReviewIssue(TypedDict):
category: str # "fact" | "logic" | "language"
severity: str # "major" | "minor"
location: str
issue: str
suggestion: str
class ReviewResult(TypedDict):
verdict: str # "pass" | "reject"
fact_score: int
logic_score: int
language_score: int
overall_score: int
pass_threshold: int
revision_notes: List[ReviewIssue]
# ── 用户干预 ──────────────────────────────────────────────────────────────────
class UserInterventionPayload(TypedDict):
action: str
approved_material_ids: List[str]
extra_keywords: List[str]
edited_outline: Optional[Outline]
outline_feedback: str
user_revision_notes: str
override_verdict: bool
# ── 主状态 ────────────────────────────────────────────────────────────────────
WritingStatus = Literal[
"researching", "awaiting_material_confirm",
"writing_outline", "awaiting_outline_confirm",
"writing_draft", "awaiting_section_help", "awaiting_draft_confirm",
"reviewing", "awaiting_review_confirm",
"revising", "done", "error",
]
class BlockedSection(TypedDict):
"""严格模式下素材不足、无法写作的章节。"""
section_title: str
reason: str
class AIWritingState(TypedDict):
# 写作配置
user_intent: str
article_type: str
target_audience: str
word_count_target: int
# 素材检索关键词数量(用户指定,1~10,默认 6)
keyword_count: int
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
writing_mode: str
# Agent A 输出
material_package: Optional[MaterialPackage]
# Agent B 输出
current_outline: Optional[Outline]
drafts: Annotated[List[DraftArticle], operator.add]
current_draft: Optional[DraftArticle]
# Agent C 输出
review_result: Optional[ReviewResult]
# 用户干预数据(每个干预点会覆盖)
last_user_intervention: Optional[UserInterventionPayload]
# 严格模式下素材不足、待用户决策的章节(写作节点产出,求助暂停点消费)
blocked_sections: Optional[List[BlockedSection]]
# 流程控制
revision_count: int
max_revisions: int
status: WritingStatus
# SSE 进度推送队列(每个节点追加)
progress_events: Annotated[List[dict], operator.add]
@@ -5,7 +5,7 @@ import logging
from langchain.agents import create_agent
from langchain_core.runnables import RunnableConfig
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
from deerflow.agents.canvas_agent.prompt import build_canvas_system_prompt
from deerflow.agents.canvas_agent.tools import update_artifact_tool
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
@@ -49,7 +49,19 @@ def make_canvas_agent(config: RunnableConfig):
if thinking_enabled and model_config and not model_config.supports_thinking:
thinking_enabled = False
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
canvas_artifact_title = cfg.get("canvas_artifact_title") or None
canvas_artifact_type = cfg.get("canvas_artifact_type") or None
system_prompt = build_canvas_system_prompt(
artifact_title=canvas_artifact_title,
artifact_type=canvas_artifact_type,
)
logger.info(
"Create CanvasAgent -> model_name: %s, thinking_enabled: %s, artifact_title: %s",
model_name,
thinking_enabled,
canvas_artifact_title,
)
middlewares = [
DanglingToolCallMiddleware(),
@@ -63,6 +75,6 @@ def make_canvas_agent(config: RunnableConfig):
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
tools=[update_artifact_tool, ask_clarification_tool],
middleware=middlewares,
system_prompt=CANVAS_SYSTEM_PROMPT,
system_prompt=system_prompt,
state_schema=ThreadState,
)
@@ -1,4 +1,18 @@
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
def build_canvas_system_prompt(
artifact_title: str | None = None,
artifact_type: str | None = None,
) -> str:
existing_artifact_block = ""
if artifact_title:
type_label = "代码" if artifact_type == "code" else "文档"
existing_artifact_block = f"""
<existing_artifact>
用户当前有一个{type_label}制品,标题为 "{artifact_title}"。
请继续编辑该制品——除非用户明确要求新建,否则不要创建新制品。
</existing_artifact>
"""
return f"""You are a Canvas AI assistant that helps users create and edit documents and code.
<canvas_behavior>
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
@@ -10,11 +24,53 @@ Rules:
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
</canvas_behavior>
<clarification_system>
Before creating a brand-new artifact or performing a major rewrite, verify you have enough
information to do the job well. When key details are missing, call `ask_clarification` FIRST
— do NOT start generating the artifact and ask questions mid-way.
**MUST clarify when (no existing artifact or major rewrite requested):**
1. OUTPUT TYPE IS VAGUE
- "帮我做个主页" / "make a homepage" → ask: tech stack (plain HTML / React / Vue?),
design style, which sections to include
- "写个报告" / "write a report" → ask: topic, target audience, expected length,
required sections
- "写个脚本" / "write a script" → ask: programming language, what it should do
2. REQUIRED CONTENT IS MISSING
- User wants a personal profile page but hasn't provided their name / bio / links
- User wants a data table but hasn't provided the data or schema
3. AMBIGUOUS APPROACH WITH SIGNIFICANTLY DIFFERENT OUTCOMES
- "Design a chat system" → ask: client–server or peer-to-peer, persistence needed?
- "Write an intro" → ask: formal essay or casual blog post?
**DO NOT clarify for:**
- Simple edits to an existing artifact: "make it bold", "fix the typo", "shorten paragraph 2"
- Formatting / style tweaks: "add comments", "use camelCase", "format the code"
- Quick actions: expand / shorten / formalize / casualize / add_comments / fix_bugs / optimize_code
- Requests where intent is clear from context (e.g., user pastes content and says "translate this")
- Follow-up turns where enough context already exists in the conversation
**Format for clarification (use options when practical):**
ask_clarification(
question="你希望这个主页使用什么技术栈?",
clarification_type="missing_info",
options=[
{{"id": "html", "label": "纯 HTML / CSS / JS", "description": "无框架,最简单"}},
{{"id": "react", "label": "React", "description": "组件化,适合复杂交互"}},
{{"id": "vue", "label": "Vue", "description": "渐进式框架,上手容易"}},
],
allow_custom=True,
)
CRITICAL: clarification ALWAYS comes before generating the artifact. Never produce a partial
artifact and then ask for clarification in the same response.
</clarification_system>
<artifact_context>
The current artifact state is injected into your context. When editing, build on the existing content exactly.
When creating from scratch, produce a complete, high-quality artifact in one shot.
</artifact_context>
{existing_artifact_block}
<selection_rewrite>
When the user's message contains a <selection> block, you MUST:
1. Identify the exact selection in the current artifact content.
@@ -38,3 +94,7 @@ When the user triggers a quick action, apply it to the full artifact unless spec
- optimize_code: Improve performance and readability
</quick_actions>
"""
# Backward-compatible constant for callers that don't pass artifact context.
CANVAS_SYSTEM_PROMPT = build_canvas_system_prompt()
@@ -5,7 +5,6 @@ from langchain.agents.middleware import AgentMiddleware
from langchain_core.runnables import RunnableConfig
from deerflow.agents.lead_agent.prompt import apply_prompt_template
from deerflow.agents.memory.summarization_hook import memory_flush_hook
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
@@ -92,9 +91,8 @@ def _create_summarization_middleware(*, app_config: AppConfig | None = None) ->
if config.summary_prompt is not None:
kwargs["summary_prompt"] = config.summary_prompt
# Memory V2:不再有后台抽取队列,无需 summarization 前的 flush hook。
hooks: list[BeforeSummarizationHook] = []
if resolved_app_config.memory.enabled:
hooks.append(memory_flush_hook)
# The logic below relies on two assumptions holding true: this factory is
# the sole entry point for DeerFlowSummarizationMiddleware, and the runtime
@@ -396,7 +394,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
)
logger.info(
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s, subagent_mode: %s",
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s",
agent_id or "default",
thinking_enabled,
reasoning_effort,
@@ -405,7 +403,6 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
subagent_enabled,
max_concurrent_subagents,
writing_mode,
subagent_mode,
)
# Inject run metadata for LangSmith trace tagging
@@ -432,7 +429,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
"subagent_enabled": subagent_enabled,
"writing_mode": writing_mode,
"writing_artifact_path": writing_artifact_path,
"tool_groups": subagent_tool_groups if subagent_mode else (agent_config.tool_groups if agent_config else None),
"tool_groups": agent_config.tool_groups if agent_config else None,
"available_skills": runtime_available_skills,
}
)
@@ -483,7 +480,6 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
writing_artifact_path=writing_artifact_path,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
soul_override=subagent_soul if subagent_mode else None,
),
state_schema=ThreadState,
)
@@ -603,32 +603,38 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
"""Get memory context for injection into system prompt.
"""组装注入系统提示词的记忆上下文 (Memory V2)。
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
本地 provider —— 返回 USER.md / MEMORY.md 的 frozen snapshot。系统提示词
在 agent 构建时定型,因此天然就是 frozen snapshot 语义。
Args:
agent_name: If provided, loads per-agent memory. If None, loads global memory.
app_config: Explicit application config. When provided, memory options
are read from this value instead of the global config singleton.
agent_name: agent 标识;``None`` 时按 "default" 处理。
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
Returns:
Formatted memory context string wrapped in XML tags, or empty string if disabled.
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
"""
try:
from deerflow.agents.memory import format_memory_for_injection, get_memory_data
from deerflow.agents.memory.manager import build_memory_manager
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.runtime.user_context import get_effective_user_id
if app_config is None:
from deerflow.config.memory_config import get_memory_config
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
config = get_memory_config()
else:
config = app_config.memory
if not config.enabled or not config.injection_enabled:
if not config.enabled or not config.injection.enabled:
return ""
memory_data = get_memory_data(agent_name, user_id=get_effective_user_id())
memory_content = format_memory_for_injection(memory_data, max_tokens=config.max_injection_tokens)
manager = build_memory_manager(
user_id=user_id,
agent_id=agent_name or "default",
config=config,
)
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
memory_content = manager.build_system_prompt()
if not memory_content.strip():
return ""
@@ -1,57 +1,37 @@
"""Memory module for DeerFlow.
This module provides a global memory mechanism that:
- Stores user context and conversation history in memory.json
- Uses LLM to summarize and extract facts from conversations
- Injects relevant memory into system prompts for personalized responses
"""
from deerflow.agents.memory.prompt import (
FACT_EXTRACTION_PROMPT,
MEMORY_UPDATE_PROMPT,
format_conversation_for_update,
format_memory_for_injection,
)
from deerflow.agents.memory.queue import (
ConversationContext,
MemoryUpdateQueue,
get_memory_queue,
reset_memory_queue,
)
from deerflow.agents.memory.storage import (
FileMemoryStorage,
MemoryStorage,
get_memory_storage,
)
from deerflow.agents.memory.updater import (
MemoryUpdater,
clear_memory_data,
delete_memory_fact,
get_memory_data,
reload_memory_data,
update_memory_from_conversation,
)
__all__ = [
# Prompt utilities
"MEMORY_UPDATE_PROMPT",
"FACT_EXTRACTION_PROMPT",
"format_memory_for_injection",
"format_conversation_for_update",
# Queue
"ConversationContext",
"MemoryUpdateQueue",
"get_memory_queue",
"reset_memory_queue",
# Storage
"MemoryStorage",
"FileMemoryStorage",
"get_memory_storage",
# Updater
"MemoryUpdater",
"clear_memory_data",
"delete_memory_fact",
"get_memory_data",
"reload_memory_data",
"update_memory_from_conversation",
]
"""DeerFlow 记忆模块 (Memory V2)。
架构 (详见 docs/MEMORY_V2_DESIGN_ZH.md):
- :class:`MemoryProvider` 可插拔记忆 provider 抽象基类
- :class:`MemoryManager` 编排内建 + 至多 1 个外部 provider
- :func:`build_memory_manager` 按生效配置组装 manager 的统一入口
- :class:`BuiltinFileProvider` 本地 USER.md / MEMORY.md provider
- :class:`HindsightProvider` Hindsight HTTP 外部 provider
- :mod:`deerflow.agents.memory.security` 记忆内容安全扫描
记忆有两条自动写入路径:LLM 主动调 ``memory`` / ``hindsight_*`` 工具,
以及 Hindsight 的 auto-retain。注入走系统提示词 (builtin frozen snapshot)
与 ``MemoryMiddleware.wrap_model_call`` (Hindsight 动态召回)。
"""
from deerflow.agents.memory.manager import MemoryManager, build_memory_manager
from deerflow.agents.memory.message_processing import filter_messages_for_memory
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.agents.memory.providers.hindsight import HindsightProvider
from deerflow.agents.memory.security import (
MemoryContentBlocked,
assert_memory_content_safe,
scan_memory_content,
)
__all__ = [
"MemoryProvider",
"MemoryManager",
"build_memory_manager",
"BuiltinFileProvider",
"HindsightProvider",
"MemoryContentBlocked",
"scan_memory_content",
"assert_memory_content_safe",
"filter_messages_for_memory",
]
@@ -0,0 +1,277 @@
"""Memory V2 - MemoryManager 编排层。
职责
====
编排 1 个内建 provider (``builtin``,始终存在) + 至多 1 个外部 provider
(P2 期的 ``hindsight``)。把系统提示词组装、召回、生命周期 hook 分发到各 provider。
任一 provider 出错都不会阻断其他 provider,也不会阻断对话。
典型用法 (由 MemoryMiddleware 每轮构建)
=======================================
::
manager = MemoryManager()
manager.add_provider(BuiltinFileProvider(...))
manager.add_provider(hindsight_provider) # 可选,至多 1 个
manager.initialize_all(user_id=..., agent_id=..., thread_id=..., base_dir=...)
block = manager.build_injection_block(query=user_msg, thread_id=tid)
# ... 把 block 注入系统提示词 ...
manager.sync_all(user_msg, assistant_msg, thread_id=tid)
"""
from __future__ import annotations
import logging
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
logger = logging.getLogger(__name__)
#: 内建 provider 的固定名字。
BUILTIN_PROVIDER_NAME = "builtin"
class MemoryManager:
"""编排内建 provider + 至多 1 个外部 provider。"""
def __init__(self) -> None:
self._providers: list[MemoryProvider] = []
self._has_external: bool = False
# ---- 注册 ------------------------------------------------------------
def add_provider(self, provider: MemoryProvider) -> bool:
"""注册一个 provider。
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
第二个会被拒绝并 log 警告。返回是否注册成功。
"""
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
if not is_builtin:
if self._has_external:
existing = next(
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
"unknown",
)
logger.warning(
"拒绝注册记忆 provider '%s' —— 已存在外部 provider '%s'。"
"同一时间只允许 1 个外部 provider,请用 config.memory.provider 选择。",
provider.name,
existing,
)
return False
self._has_external = True
self._providers.append(provider)
logger.info("记忆 provider '%s' 已注册", provider.name)
return True
@property
def providers(self) -> list[MemoryProvider]:
"""按注册顺序返回所有 provider。"""
return list(self._providers)
def get_provider(self, name: str) -> MemoryProvider | None:
"""按名字取 provider,不存在返回 ``None``。"""
for p in self._providers:
if p.name == name:
return p
return None
# ---- 生命周期 ---------------------------------------------------------
def initialize_all(self, **kwargs: Any) -> None:
"""初始化所有 provider。单个 provider 失败不影响其他。"""
for provider in self._providers:
try:
provider.initialize(**kwargs)
except Exception as exc: # noqa: BLE001 - provider 失败必须隔离
logger.warning("记忆 provider '%s' initialize 失败:%s", provider.name, exc)
def shutdown_all(self) -> None:
"""关闭所有 provider (逆序,保证干净拆除)。"""
for provider in reversed(self._providers):
try:
provider.shutdown()
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' shutdown 失败:%s", provider.name, exc)
# ---- 系统提示词组装 / 召回 -------------------------------------------
def build_system_prompt(self) -> str:
"""收集所有 provider 的静态系统提示词块。"""
blocks: list[str] = []
for provider in self._providers:
try:
block = provider.system_prompt_block()
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' system_prompt_block 失败:%s", provider.name, exc)
continue
if block and block.strip():
blocks.append(block)
return "\n\n".join(blocks)
def prefetch_all(self, query: str, *, thread_id: str = "") -> str:
"""收集所有 provider 的召回上下文。单个失败不阻断其他。"""
parts: list[str] = []
for provider in self._providers:
try:
result = provider.prefetch(query, thread_id=thread_id)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' prefetch 失败 (非致命):%s", provider.name, exc)
continue
if result and result.strip():
parts.append(result)
return "\n\n".join(parts)
def build_injection_block(self, query: str, *, thread_id: str = "") -> str:
"""组装注入系统提示词的完整记忆块。
= 所有 provider 的静态块 (frozen snapshot 等) + 动态召回块。
返回空串表示无内容可注入。
"""
static_part = self.build_system_prompt()
dynamic_part = self.prefetch_all(query, thread_id=thread_id)
parts = [p for p in (static_part, dynamic_part) if p and p.strip()]
return "\n\n".join(parts)
def queue_prefetch_all(self, query: str, *, thread_id: str = "") -> None:
"""通知所有 provider 后台预热下一轮召回。"""
for provider in self._providers:
try:
provider.queue_prefetch(query, thread_id=thread_id)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' queue_prefetch 失败:%s", provider.name, exc)
# ---- 写入 ------------------------------------------------------------
def sync_all(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
turn_index: int | None = None,
) -> None:
"""把一轮完成的对话同步到所有 provider。
``turn_index`` 是当前对话的轮次序号,透传给各 provider 用于轮次节流。
"""
for provider in self._providers:
try:
provider.sync_turn(
user_content,
assistant_content,
thread_id=thread_id,
turn_index=turn_index,
)
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' sync_turn 失败:%s", provider.name, exc)
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入后,通知外部 provider 镜像。
跳过 builtin 自身 (它就是写入源,镜像回去没有意义)。
"""
for provider in self._providers:
if provider.name == BUILTIN_PROVIDER_NAME:
continue
try:
provider.on_memory_write(action, target, content, metadata)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' on_memory_write 失败:%s", provider.name, exc)
# ---- 可选 hooks ------------------------------------------------------
def on_session_switch_all(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""通知所有 provider thread_id 已切换。"""
if not new_thread_id:
return
for provider in self._providers:
try:
provider.on_session_switch(
new_thread_id,
parent_thread_id=parent_thread_id,
reset=reset,
**kwargs,
)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' on_session_switch 失败:%s", provider.name, exc)
def build_memory_manager(
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
config: Any = None,
) -> MemoryManager:
"""按生效配置组装并初始化一个 MemoryManager。
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
所有 provider 已 ``initialize`` 完毕,可直接使用。
Parameters
----------
user_id / agent_id / thread_id:
请求上下文标识。
config:
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.config.paths import get_paths
if config is None:
config = get_effective_memory_config(user_id)
manager = MemoryManager()
if config.builtin.enabled:
manager.add_provider(
BuiltinFileProvider(
memory_char_limit=config.builtin.memory_char_limit,
user_char_limit=config.builtin.user_char_limit,
deduplicate_on_load=config.builtin.deduplicate_on_load,
security=config.security,
)
)
if config.provider == "hindsight":
try:
from deerflow.agents.memory.providers.hindsight import HindsightProvider
hindsight = HindsightProvider(config.hindsight)
if hindsight.is_available():
manager.add_provider(hindsight)
else:
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
manager.initialize_all(
user_id=user_id,
agent_id=agent_id or "default",
thread_id=thread_id,
base_dir=str(get_paths().base_dir),
)
return manager
__all__ = ["MemoryManager", "BUILTIN_PROVIDER_NAME", "build_memory_manager"]
@@ -1,4 +1,8 @@
"""Shared helpers for turning conversations into memory update inputs."""
"""把对话整理成记忆写入所需输入的共享辅助函数。
Memory V2 说明:后台抽取已舍弃,这里只保留 ``filter_messages_for_memory``
(供 MemoryMiddleware 取出最后一轮对话用于 Hindsight retain)。
"""
from __future__ import annotations
@@ -7,34 +11,6 @@ from copy import copy
from typing import Any
_UPLOAD_BLOCK_RE = re.compile(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", re.IGNORECASE)
_CORRECTION_PATTERNS = (
re.compile(r"\bthat(?:'s| is) (?:wrong|incorrect)\b", re.IGNORECASE),
re.compile(r"\byou misunderstood\b", re.IGNORECASE),
re.compile(r"\btry again\b", re.IGNORECASE),
re.compile(r"\bredo\b", re.IGNORECASE),
re.compile(r"不对"),
re.compile(r"你理解错了"),
re.compile(r"你理解有误"),
re.compile(r"重试"),
re.compile(r"重新来"),
re.compile(r"换一种"),
re.compile(r"改用"),
)
_REINFORCEMENT_PATTERNS = (
re.compile(r"\byes[,.]?\s+(?:exactly|perfect|that(?:'s| is) (?:right|correct|it))\b", re.IGNORECASE),
re.compile(r"\bperfect(?:[.!?]|$)", re.IGNORECASE),
re.compile(r"\bexactly\s+(?:right|correct)\b", re.IGNORECASE),
re.compile(r"\bthat(?:'s| is)\s+(?:exactly\s+)?(?:right|correct|what i (?:wanted|needed|meant))\b", re.IGNORECASE),
re.compile(r"\bkeep\s+(?:doing\s+)?that\b", re.IGNORECASE),
re.compile(r"\bjust\s+(?:like\s+)?(?:that|this)\b", re.IGNORECASE),
re.compile(r"\bthis is (?:great|helpful)\b(?:[.!?]|$)", re.IGNORECASE),
re.compile(r"\bthis is what i wanted\b(?:[.!?]|$)", re.IGNORECASE),
re.compile(r"对[,,]?\s*就是这样(?:[。!?!?.]|$)"),
re.compile(r"完全正确(?:[。!?!?.]|$)"),
re.compile(r"(?:对[,,]?\s*)?就是这个意思(?:[。!?!?.]|$)"),
re.compile(r"正是我想要的(?:[。!?!?.]|$)"),
re.compile(r"继续保持(?:[。!?!?.]|$)"),
)
def extract_message_text(message: Any) -> str:
@@ -53,6 +29,16 @@ def extract_message_text(message: Any) -> str:
return str(content)
def count_user_turns(filtered_messages: list[Any]) -> int:
"""统计已过滤消息里的用户轮次数 (= human 消息条数)。
供 MemoryMiddleware 算出当前轮次序号传给 provider 做轮次节流 ——
provider 每轮被重建、无法靠自身计数,必须由这里按对话历史推算。
入参应是 :func:`filter_messages_for_memory` 的输出。
"""
return sum(1 for msg in filtered_messages if getattr(msg, "type", None) == "human")
def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
"""Keep only user inputs and final assistant responses for memory updates."""
filtered = []
@@ -83,27 +69,3 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
filtered.append(msg)
return filtered
def detect_correction(messages: list[Any]) -> bool:
"""Detect explicit user corrections in recent conversation turns."""
recent_user_msgs = [msg for msg in messages[-6:] if getattr(msg, "type", None) == "human"]
for msg in recent_user_msgs:
content = extract_message_text(msg).strip()
if content and any(pattern.search(content) for pattern in _CORRECTION_PATTERNS):
return True
return False
def detect_reinforcement(messages: list[Any]) -> bool:
"""Detect explicit positive reinforcement signals in recent conversation turns."""
recent_user_msgs = [msg for msg in messages[-6:] if getattr(msg, "type", None) == "human"]
for msg in recent_user_msgs:
content = extract_message_text(msg).strip()
if content and any(pattern.search(content) for pattern in _REINFORCEMENT_PATTERNS):
return True
return False
@@ -1,363 +0,0 @@
"""Prompt templates for memory update and injection."""
import math
import re
from typing import Any
try:
import tiktoken
TIKTOKEN_AVAILABLE = True
except ImportError:
TIKTOKEN_AVAILABLE = False
# Prompt template for updating memory based on conversation
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
Current Memory State:
<current_memory>
{current_memory}
</current_memory>
New Conversation to Process:
<conversation>
{conversation}
</conversation>
Instructions:
1. Analyze the conversation for important information about the user
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
3. Update the memory sections as needed following the detailed length guidelines below
Before extracting facts, perform a structured reflection on the conversation:
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
If yes, record them as facts with the most appropriate category and confidence.
{correction_hint}
Memory Section Guidelines:
**User Context** (Current state - concise summaries):
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
Example: Core contributor, project names with metrics (16k+ stars), technical stack
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
Example: Bilingual capabilities, specific interest areas, expertise domains
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
Include: Active implementation work, troubleshooting issues, market/research interests
Note: This captures SEVERAL concurrent focus areas, not just one task
**History** (Temporal context - rich paragraphs):
- recentMonths: Detailed summary of recent activities (4-6 sentences or 1-2 paragraphs)
Timeline: Last 1-3 months of interactions
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
- earlierContext: Important historical patterns (3-5 sentences or 1 paragraph)
Timeline: 3-12 months ago
Include: Past projects, learning journeys, established patterns
- longTermBackground: Persistent background and foundational context (2-4 sentences)
Timeline: Overall/foundational information
Include: Core expertise, longstanding interests, fundamental working style
**Facts Extraction**:
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
- Include proper nouns (company names, project names, technology names)
- Preserve technical terminology and version numbers
- Categories:
* preference: Tools, styles, approaches user prefers/dislikes
* knowledge: Specific expertise, technologies mastered, domain knowledge
* context: Background facts (job title, projects, locations, languages)
* behavior: Working patterns, communication habits, problem-solving approaches
* goal: Stated objectives, learning targets, project ambitions
* correction: Explicit agent mistakes or user corrections, including the correct approach
- Confidence levels:
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
* 0.7-0.8: Strongly implied from actions/discussions
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
**What Goes Where**:
- workContext: Current job, active projects, primary tech stack
- personalContext: Languages, personality, interests outside direct work tasks
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
- recentMonths: Detailed account of recent technical explorations and work
- earlierContext: Patterns from slightly older interactions still relevant
- longTermBackground: Unchanging foundational facts about the user
**Multilingual Content**:
- Preserve original language for proper nouns and company names
- Keep technical terms in their original form (DeepSeek, LangGraph, etc.)
- Note language capabilities in personalContext
Output Format (JSON):
{{
"user": {{
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
}},
"history": {{
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
}},
"newFacts": [
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
],
"factsToRemove": ["fact_id_1", "fact_id_2"]
}}
Important Rules:
- Only set shouldUpdate=true if there's meaningful new information
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
- Include specific metrics, version numbers, and proper nouns in facts
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
- Remove facts that are contradicted by new information
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
Keep 3-5 concurrent focus themes that are still active and relevant
- For history sections, integrate new information chronologically into appropriate time period
- Preserve technical accuracy - keep exact names of technologies, companies, projects
- Focus on information useful for future interactions and personalization
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
session-specific and ephemeral — they will not be accessible in future sessions.
Recording upload events causes confusion in subsequent conversations.
Return ONLY valid JSON, no explanation or markdown."""
# Prompt template for extracting facts from a single message
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
Message:
{message}
Extract facts in this JSON format:
{{
"facts": [
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
]
}}
Categories:
- preference: User preferences (likes/dislikes, styles, tools)
- knowledge: User's expertise or knowledge areas
- context: Background context (location, job, projects)
- behavior: Behavioral patterns
- goal: User's goals or objectives
- correction: Explicit corrections or mistakes to avoid repeating
Rules:
- Only extract clear, specific facts
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
- Skip vague or temporary information
Return ONLY valid JSON."""
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
"""Count tokens in text using tiktoken.
Args:
text: The text to count tokens for.
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
Returns:
The number of tokens in the text.
"""
if not TIKTOKEN_AVAILABLE:
# Fallback to character-based estimation if tiktoken is not available
return len(text) // 4
try:
encoding = tiktoken.get_encoding(encoding_name)
return len(encoding.encode(text))
except Exception:
# Fallback to character-based estimation on error
return len(text) // 4
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
"""Coerce a confidence-like value to a bounded float in [0, 1].
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
to the default before clamping, preventing them from dominating ranking.
The ``default`` parameter is assumed to be a finite value.
"""
try:
confidence = float(value)
except (TypeError, ValueError):
return max(0.0, min(1.0, default))
if not math.isfinite(confidence):
return max(0.0, min(1.0, default))
return max(0.0, min(1.0, confidence))
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
"""Format memory data for injection into system prompt.
Args:
memory_data: The memory data dictionary.
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
Returns:
Formatted memory string for system prompt injection.
"""
if not memory_data:
return ""
sections = []
# Format user context
user_data = memory_data.get("user", {})
if user_data:
user_sections = []
work_ctx = user_data.get("workContext", {})
if work_ctx.get("summary"):
user_sections.append(f"Work: {work_ctx['summary']}")
personal_ctx = user_data.get("personalContext", {})
if personal_ctx.get("summary"):
user_sections.append(f"Personal: {personal_ctx['summary']}")
top_of_mind = user_data.get("topOfMind", {})
if top_of_mind.get("summary"):
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
if user_sections:
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
# Format history
history_data = memory_data.get("history", {})
if history_data:
history_sections = []
recent = history_data.get("recentMonths", {})
if recent.get("summary"):
history_sections.append(f"Recent: {recent['summary']}")
earlier = history_data.get("earlierContext", {})
if earlier.get("summary"):
history_sections.append(f"Earlier: {earlier['summary']}")
background = history_data.get("longTermBackground", {})
if background.get("summary"):
history_sections.append(f"Background: {background['summary']}")
if history_sections:
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
# Format facts (sorted by confidence; include as many as token budget allows)
facts_data = memory_data.get("facts", [])
if isinstance(facts_data, list) and facts_data:
ranked_facts = sorted(
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
reverse=True,
)
# Compute token count for existing sections once, then account
# incrementally for each fact line to avoid full-string re-tokenization.
base_text = "\n\n".join(sections)
base_tokens = _count_tokens(base_text) if base_text else 0
# Account for the separator between existing sections and the facts section.
facts_header = "Facts:\n"
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
running_tokens = base_tokens + separator_tokens
fact_lines: list[str] = []
for fact in ranked_facts:
content_value = fact.get("content")
if not isinstance(content_value, str):
continue
content = content_value.strip()
if not content:
continue
category = str(fact.get("category", "context")).strip() or "context"
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
source_error = fact.get("sourceError")
if category == "correction" and isinstance(source_error, str) and source_error.strip():
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
else:
line = f"- [{category} | {confidence:.2f}] {content}"
# Each additional line is preceded by a newline (except the first).
line_text = ("\n" + line) if fact_lines else line
line_tokens = _count_tokens(line_text)
if running_tokens + line_tokens <= max_tokens:
fact_lines.append(line)
running_tokens += line_tokens
else:
break
if fact_lines:
sections.append("Facts:\n" + "\n".join(fact_lines))
if not sections:
return ""
result = "\n\n".join(sections)
# Use accurate token counting with tiktoken
token_count = _count_tokens(result)
if token_count > max_tokens:
# Truncate to fit within token limit
# Estimate characters to remove based on token ratio
char_per_token = len(result) / token_count
target_chars = int(max_tokens * char_per_token * 0.95) # 95% to leave margin
result = result[:target_chars] + "\n..."
return result
def format_conversation_for_update(messages: list[Any]) -> str:
"""Format conversation messages for memory update prompt.
Args:
messages: List of conversation messages.
Returns:
Formatted conversation string.
"""
lines = []
for msg in messages:
role = getattr(msg, "type", "unknown")
content = getattr(msg, "content", str(msg))
# Handle content that might be a list (multimodal)
if isinstance(content, list):
text_parts = []
for p in content:
if isinstance(p, str):
text_parts.append(p)
elif isinstance(p, dict):
text_val = p.get("text")
if isinstance(text_val, str):
text_parts.append(text_val)
content = " ".join(text_parts) if text_parts else str(content)
# Strip uploaded_files tags from human messages to avoid persisting
# ephemeral file path info into long-term memory. Skip the turn entirely
# when nothing remains after stripping (upload-only message).
if role == "human":
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
if not content:
continue
# Truncate very long messages
if len(str(content)) > 1000:
content = str(content)[:1000] + "..."
if role == "human":
lines.append(f"User: {content}")
elif role == "ai":
lines.append(f"Assistant: {content}")
return "\n\n".join(lines)
@@ -0,0 +1,205 @@
"""Memory V2 - 记忆 Provider 抽象基类。
设计动机
========
Hermes Agent 的可插拔记忆系统的 deerflow 版本。所有具体的记忆后端
(builtin 本地文件、Hindsight HTTP 等)都实现这个接口,由 ``MemoryManager``
统一编排。Provider 之间互不感知,可自由组合 (内建 + 至多 1 个外部 plugin)。
生命周期
========
:func:`MemoryProvider.initialize` 初始化 (创建 bank、加载本地文件、建立连接)
:func:`MemoryProvider.system_prompt_block` 注入系统提示词的静态说明文本
:func:`MemoryProvider.prefetch` 每轮对话前召回上下文
:func:`MemoryProvider.queue_prefetch` 下一轮的后台预热
:func:`MemoryProvider.sync_turn` 每轮对话后异步写入后端
:func:`MemoryProvider.get_tool_schemas` 暴露给 LLM 的工具 schema
:func:`MemoryProvider.handle_tool_call` 路由工具调用
:func:`MemoryProvider.shutdown` 清理 (flush 队列、关连接)
可选 hooks
==========
仅按 ``生产推荐档`` 移植两个 ——
- :func:`MemoryProvider.on_session_switch` thread_id 切换时 (branch/resume/reset/压缩后)
- :func:`MemoryProvider.on_memory_write` builtin 写入时,通知外部 provider 镜像
参见 ``docs/MEMORY_V2_DESIGN_ZH.md`` §4 的完整接口定义和 §16 的实施分期。
"""
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Any
logger = logging.getLogger(__name__)
class MemoryProvider(ABC):
"""所有记忆 provider 必须实现的接口。"""
# ---- 身份 ------------------------------------------------------------
@property
@abstractmethod
def name(self) -> str:
"""短标识,如 ``'builtin'`` / ``'hindsight'``。"""
# ---- 生命周期 ---------------------------------------------------------
@abstractmethod
def is_available(self) -> bool:
"""是否已配置并可用。
在 agent 启动时被调用,用于判断要不要激活该 provider。
实现里只检查配置和依赖,不发网络请求。
"""
@abstractmethod
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str,
**kwargs: Any,
) -> None:
"""初始化:打开连接、创建 bank、加载本地文件等。
kwargs 至少包含 ``user_id`` / ``base_dir``,其余字段视 provider 而定。
Provider 可以根据 ``agent_id is None`` 决定走全局还是 per-agent 路径。
Parameters
----------
user_id:
当前用户的隔离标识(无认证场景下为 ``"default"``)。
agent_id:
当前 agent 的稳定标识;``None`` 表示走全局/默认 agent。
thread_id:
当前对话的 thread id;某些 provider 用它做并发隔离或 bank 后缀。
base_dir:
``Paths.base_dir`` 字符串形式,避免每个 provider 重复解析路径。
"""
def shutdown(self) -> None:
"""清理:刷队列、关连接。默认 no-op。"""
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""注入系统提示词的静态文本。
典型用途:USER.md / MEMORY.md 的 frozen snapshot,provider 状态说明等。
返回空串表示不贡献内容。
"""
return ""
# ---- 召回 / 写入 -----------------------------------------------------
def prefetch(self, query: str, *, thread_id: str = "") -> str:
"""对话前召回上下文。
在每次 LLM 调用前由 :class:`MemoryMiddleware` 触发。``query`` 通常
是用户最近一条消息。实现要快;真正的网络召回应该走后台线程,
这里直接返回缓存结果。
返回值:带格式的文本块,会被拼接到系统提示词末尾。空串表示无内容。
"""
return ""
def queue_prefetch(self, query: str, *, thread_id: str = "") -> None:
"""后台预热下一轮的召回。默认 no-op。"""
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
turn_index: int | None = None,
) -> None:
"""每轮对话结束后把对话持久化到后端。
实现应当非阻塞 —— 如果后端有延迟,排队到后台处理而不是直接调用。
默认 no-op。
``turn_index`` 是当前对话的轮次序号 (从 1 开始,由编排层按对话历史算出)。
provider 通常被每轮重新构造、无法靠自身计数,需要按轮次节流写入
(例如 "每 N 轮 retain 一次") 的实现应当用它而非进程内计数器。
缺省 ``None`` 表示调用方未提供。
"""
# ---- LLM 工具 --------------------------------------------------------
@abstractmethod
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""暴露给 LLM 的工具 schema 列表。
每个 schema 遵循 OpenAI function calling 格式::
{"name": "...", "description": "...", "parameters": {...}}
空数组表示 context-only(只做自动 recall/retain,不让 LLM 看见工具)。
"""
def handle_tool_call(
self, tool_name: str, args: dict[str, Any], **kwargs: Any
) -> str:
"""处理一个工具调用,必须返回 JSON 字符串。
``MemoryManager`` 会确保只把本 provider ``get_tool_schemas`` 里
声明的工具名路由到这里。
"""
raise NotImplementedError(
f"Provider {self.name} does not handle tool {tool_name}"
)
# ---- 可选 hooks (生产推荐档移植) -------------------------------------
def on_session_switch(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""thread_id 切换时触发。
deerflow 的 thread_id 在以下场景会变化:
- 用户新建/切换 thread (``reset=True``)
- LangGraph thread branch (``reset=False``,``parent_thread_id`` 为原 thread)
- SummarizationMiddleware 压缩后继续 (``reset=False``)
Provider 在 ``initialize`` 时如果缓存了 per-session 的状态
(``_session_turns`` 累积缓冲、bank 关联、查询缓存等),应当在
这里刷新。默认 no-op。
"""
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入时触发,通知外部 provider 镜像同一条记忆。
Parameters
----------
action:
``'add'`` / ``'replace'`` / ``'remove'``。
target:
``'user'`` / ``'memory'``,决定走全局 user bank 还是 agent 工作 bank。
content:
条目内容。``remove`` 时为待删除条目的内容(供 external 做幂等)。
metadata:
写入来源元数据 (``user_id`` / ``agent_id`` / ``thread_id`` / ``tool_name``)。
默认 no-op —— 只有需要镜像 builtin 写入的外部 provider (例如 Hindsight)
才需要重写。``MemoryManager`` 调度时会跳过 builtin 自身,避免无意义的回环。
"""
__all__ = ["MemoryProvider"]
@@ -0,0 +1,10 @@
"""Memory V2 - 具体 provider 实现。
- :class:`~deerflow.agents.memory.providers.builtin.BuiltinFileProvider`
本地文件 provider (USER.md / MEMORY.md),始终启用。
- ``HindsightProvider`` — Hindsight HTTP 客户端,P2 期接入。
"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
__all__ = ["BuiltinFileProvider"]
@@ -0,0 +1,417 @@
"""Memory V2 - 本地文件 provider (USER.md / MEMORY.md)。
从 Hermes ``tools/memory_tool.py`` 的 ``MemoryStore`` 移植,适配 deerflow 的
per-user / per-agent 目录结构与 :class:`MemoryProvider` 接口。
两类存储
========
- ``USER.md`` 用户画像 —— LLM 主动写入的 ``target="user"`` 记忆,跨 agent 共享。
路径:``{base_dir}/users/{user_id}/USER.md``
- ``MEMORY.md`` agent 工作笔记 —— LLM 主动写入的 ``target="memory"`` 记忆,agent 隔离。
路径:``{base_dir}/users/{user_id}/agents/{agent_id}/MEMORY.md``
设计要点
========
- 条目以 ``\\n§\\n`` 分隔,支持多行;采用字符级配额 (与模型无关)。
- frozen snapshot:``initialize`` 时读盘一次并冻结为系统提示词块,会话内写入
直接落盘 (durable) 但不刷新快照 —— 保持系统提示词稳定,利于 prefix cache。
- 读-改-写经文件锁串行化;写入用临时文件 + 原子 rename,避免并发读到半截文件。
- 写入前经 :mod:`deerflow.agents.memory.security` 扫描注入/外渗。
"""
from __future__ import annotations
import logging
import os
import tempfile
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.agents.memory.security import scan_memory_content
from deerflow.config.memory_config import SecurityConfig
from deerflow.config.paths import get_paths
from deerflow.runtime.user_context import DEFAULT_USER_ID
logger = logging.getLogger(__name__)
# 跨平台文件锁:Unix 用 fcntl,Windows 用 msvcrt。
_fcntl: Any = None
_msvcrt: Any = None
try: # pragma: no cover - 平台相关
import fcntl as _fcntl_mod
_fcntl = _fcntl_mod
except ImportError: # pragma: no cover - Windows
try:
import msvcrt as _msvcrt_mod
_msvcrt = _msvcrt_mod
except ImportError:
pass
#: 条目分隔符。``§`` = section sign (U+00A7)。
ENTRY_DELIMITER = "\n§\n"
#: target 取值。
_VALID_TARGETS = ("memory", "user")
# 同一进程内对同一文件路径的写入串行化锁 (文件锁之上再加一层,降低自争用)。
_PATH_LOCKS: dict[str, threading.Lock] = {}
_PATH_LOCKS_GUARD = threading.Lock()
def _get_path_lock(path: Path) -> threading.Lock:
"""返回某文件路径对应的进程内锁 (按路径字符串缓存)。"""
key = str(path)
with _PATH_LOCKS_GUARD:
lock = _PATH_LOCKS.get(key)
if lock is None:
lock = threading.Lock()
_PATH_LOCKS[key] = lock
return lock
@contextmanager
def _file_lock(path: Path):
"""获取某文件的排他锁,保证读-改-写安全。
用单独的 ``.lock`` 文件加锁,这样记忆文件本身仍可被 ``os.replace`` 原子替换。
进程内先抢 :func:`_get_path_lock`,再抢跨进程文件锁。
"""
process_lock = _get_path_lock(path)
process_lock.acquire()
lock_path = path.with_suffix(path.suffix + ".lock")
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
if _fcntl is None and _msvcrt is None: # pragma: no cover - 无锁平台
yield
return
if _msvcrt is not None and (not lock_path.exists() or lock_path.stat().st_size == 0):
lock_path.write_text(" ", encoding="utf-8")
fd = open(lock_path, "r+" if _msvcrt is not None else "a+")
try:
if _fcntl is not None:
_fcntl.flock(fd, _fcntl.LOCK_EX)
else: # pragma: no cover - Windows
fd.seek(0)
_msvcrt.locking(fd.fileno(), _msvcrt.LK_LOCK, 1)
yield
finally:
try:
if _fcntl is not None:
_fcntl.flock(fd, _fcntl.LOCK_UN)
elif _msvcrt is not None: # pragma: no cover - Windows
fd.seek(0)
_msvcrt.locking(fd.fileno(), _msvcrt.LK_UNLCK, 1)
except OSError:
pass
fd.close()
finally:
process_lock.release()
def _read_entries(path: Path) -> list[str]:
"""读取一个记忆文件并按分隔符切成条目列表。
无需加锁:写入用原子 rename,读者总能看到旧文件或新文件的完整内容。
"""
if not path.exists():
return []
try:
raw = path.read_text(encoding="utf-8")
except OSError:
return []
if not raw.strip():
return []
entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)]
return [e for e in entries if e]
def _write_entries(path: Path, entries: list[str]) -> None:
"""用临时文件 + 原子 rename 写入记忆文件。"""
content = ENTRY_DELIMITER.join(entries) if entries else ""
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".mem_")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(content)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_path, path)
except BaseException:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
class BuiltinFileProvider(MemoryProvider):
"""本地文件记忆 provider,始终可用,不可移除。"""
def __init__(
self,
*,
memory_char_limit: int = 2200,
user_char_limit: int = 1375,
deduplicate_on_load: bool = True,
security: SecurityConfig | None = None,
) -> None:
self._memory_char_limit = memory_char_limit
self._user_char_limit = user_char_limit
self._deduplicate_on_load = deduplicate_on_load
self._security = security or SecurityConfig()
self._user_id: str = DEFAULT_USER_ID
self._agent_id: str = "default"
self._user_path: Path | None = None
self._memory_path: Path | None = None
# 实时状态 (工具写入后立即更新并落盘)
self._user_entries: list[str] = []
self._memory_entries: list[str] = []
# frozen snapshot (initialize 时冻结,系统提示词注入用)
self._snapshot: dict[str, str] = {"user": "", "memory": ""}
# ---- 身份 ------------------------------------------------------------
@property
def name(self) -> str:
return "builtin"
def is_available(self) -> bool:
"""本地文件 provider 无外部依赖,始终可用。"""
return True
# ---- 生命周期 ---------------------------------------------------------
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str = "",
**kwargs: Any,
) -> None:
"""解析 per-user / per-agent 路径,加载文件并冻结系统提示词快照。"""
self._user_id = user_id or DEFAULT_USER_ID
self._agent_id = (agent_id or "default").lower()
paths = get_paths()
self._user_path = paths.user_profile_md_file(self._user_id)
self._memory_path = paths.user_agent_memory_md_file(self._user_id, self._agent_id)
self._load_from_disk()
def _load_from_disk(self) -> None:
"""从磁盘加载两个文件,去重,并冻结系统提示词快照。"""
assert self._user_path is not None and self._memory_path is not None
self._user_entries = _read_entries(self._user_path)
self._memory_entries = _read_entries(self._memory_path)
if self._deduplicate_on_load:
self._user_entries = list(dict.fromkeys(self._user_entries))
self._memory_entries = list(dict.fromkeys(self._memory_entries))
self._snapshot = {
"user": self._render_block("user", self._user_entries),
"memory": self._render_block("memory", self._memory_entries),
}
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""返回 frozen snapshot:USER PROFILE 块 + MEMORY 块。
返回 ``initialize`` 时冻结的内容,会话内的工具写入不影响它 ——
保持系统提示词稳定,利于 prefix cache。
"""
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
return "\n\n".join(blocks)
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
不经 MemoryManager 路由,因此这里返回空列表。
"""
return []
# ---- 工具写入操作 (供 memory 工具直接调用) ---------------------------
def add(self, target: str, content: str) -> dict[str, Any]:
"""追加一条记忆条目。超出字符配额时返回错误。"""
content = (content or "").strip()
if not content:
return {"success": False, "error": "条目内容不能为空。"}
if (err := self._scan(content)) is not None:
return {"success": False, "error": err}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
if content in entries:
return self._success(target, "条目已存在,未重复添加。")
limit = self._char_limit(target)
new_total = len(ENTRY_DELIMITER.join([*entries, content]))
if new_total > limit:
current = self._char_count(entries)
return {
"success": False,
"error": (
f"记忆已用 {current:,}/{limit:,} 字符,"
f"再加这条 ({len(content)} 字符) 会超出配额。"
"请先 replace 或 remove 现有条目。"
),
"entries": entries,
}
entries.append(content)
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已添加。")
def replace(self, target: str, old_text: str, new_content: str) -> dict[str, Any]:
"""用 ``old_text`` 短串定位条目,替换为 ``new_content``。"""
old_text = (old_text or "").strip()
new_content = (new_content or "").strip()
if not old_text:
return {"success": False, "error": "old_text 不能为空。"}
if not new_content:
return {"success": False, "error": "new_content 不能为空;删除条目请用 remove。"}
if (err := self._scan(new_content)) is not None:
return {"success": False, "error": err}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
if not matches:
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
if len({e for _, e in matches}) > 1:
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
return {
"success": False,
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
"matches": previews,
}
idx = matches[0][0]
limit = self._char_limit(target)
test_entries = list(entries)
test_entries[idx] = new_content
new_total = len(ENTRY_DELIMITER.join(test_entries))
if new_total > limit:
return {
"success": False,
"error": (
f"替换后会达到 {new_total:,}/{limit:,} 字符,超出配额。"
"请缩短新内容或先删除其他条目。"
),
}
entries[idx] = new_content
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已替换。")
def remove(self, target: str, old_text: str) -> dict[str, Any]:
"""用 ``old_text`` 短串定位并删除条目。"""
old_text = (old_text or "").strip()
if not old_text:
return {"success": False, "error": "old_text 不能为空。"}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
if not matches:
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
if len({e for _, e in matches}) > 1:
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
return {
"success": False,
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
"matches": previews,
}
entries.pop(matches[0][0])
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已删除。")
def read(self, target: str) -> list[str]:
"""返回某 target 的实时条目列表 (从磁盘重读)。"""
return self._reload_target(target)
def usage(self, target: str) -> str:
"""返回某 target 的字符用量描述,如 ``'12% — 165/1375 字符'``。"""
entries = self._reload_target(target)
current = self._char_count(entries)
limit = self._char_limit(target)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
return f"{pct}% — {current:,}/{limit:,} 字符"
# ---- 内部辅助 --------------------------------------------------------
def _scan(self, content: str) -> str | None:
return scan_memory_content(
content,
scan_threats=self._security.scan_content,
block_invisible_unicode=self._security.block_invisible_unicode,
)
def _path_for(self, target: str) -> Path:
if target not in _VALID_TARGETS:
raise ValueError(f"非法 target '{target}',只能是 'memory' 或 'user'。")
if self._user_path is None or self._memory_path is None:
raise RuntimeError("BuiltinFileProvider 未初始化,请先调用 initialize()。")
return self._user_path if target == "user" else self._memory_path
def _char_limit(self, target: str) -> int:
return self._user_char_limit if target == "user" else self._memory_char_limit
@staticmethod
def _char_count(entries: list[str]) -> int:
return len(ENTRY_DELIMITER.join(entries)) if entries else 0
def _reload_target(self, target: str) -> list[str]:
"""加锁状态下从磁盘重读条目 (拿到其他会话写入的最新状态)。"""
entries = _read_entries(self._path_for(target))
if self._deduplicate_on_load:
entries = list(dict.fromkeys(entries))
self._set_entries(target, entries)
return entries
def _set_entries(self, target: str, entries: list[str]) -> None:
if target == "user":
self._user_entries = entries
else:
self._memory_entries = entries
def _success(self, target: str, message: str) -> dict[str, Any]:
entries = self._user_entries if target == "user" else self._memory_entries
current = self._char_count(entries)
limit = self._char_limit(target)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
return {
"success": True,
"target": target,
"entries": entries,
"entry_count": len(entries),
"usage": f"{pct}% — {current:,}/{limit:,} 字符",
"message": message,
}
def _render_block(self, target: str, entries: list[str]) -> str:
"""渲染一个系统提示词块 (带表头和用量指示)。"""
if not entries:
return ""
limit = self._char_limit(target)
content = ENTRY_DELIMITER.join(entries)
current = len(content)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
if target == "user":
header = f"用户画像 USER PROFILE [{pct}% — {current:,}/{limit:,} 字符]"
else:
header = f"记忆笔记 MEMORY [{pct}% — {current:,}/{limit:,} 字符]"
separator = "═" * 46
return f"{separator}\n{header}\n{separator}\n{content}"
__all__ = ["BuiltinFileProvider", "ENTRY_DELIMITER"]
@@ -0,0 +1,462 @@
"""Memory V2 - Hindsight 外部 provider。
Hindsight 是带知识图谱、实体消解、多策略检索的长期记忆服务。本 provider 以
HTTP 客户端方式接入一个用户自起的 Hindsight Docker 实例 (``local_external`` 模式)。
双 bank 隔离 (B 方案,详见 docs/MEMORY_V2_DESIGN_ZH.md §3)
==========================================================
- 全局 user bank ``bank_id_template`` 例:``deerflow-user-{user_id}``
存用户身份、偏好,跨 agent 共享。
- agent 工作 bank ``work_bank_id_template`` 例:``deerflow-work-{user_id}-{agent_id}``
存 agent 私有工作笔记。
写入路由:
- LLM 主动 ``memory(target="user")`` -> 镜像到 user bank
- LLM 主动 ``memory(target="memory")`` -> 镜像到 work bank
- 自动 retain 的对话 (``sync_turn``) -> work bank
召回:``prefetch`` 并发查两个 bank,合并结果。
document_id 与轮次计数
======================
本 provider 被 ``build_memory_manager`` 每轮重新构造 —— 是无状态的。因此:
- ``document_id`` 每次 ``initialize`` 用带时间戳的新值,每轮对话的 retain 落到
各自独立的 Hindsight 文档。
- ``retain_every_n_turns`` 的轮次判定依赖外部传入的 ``turn_index``
(由 ``MemoryMiddleware`` 按对话历史算出);未传时退回进程内 ``_turn_counter``。
客户端生命周期
==============
``_run_coro`` 每次操作新建一个事件循环。aiohttp ``ClientSession`` 是
loop-bound 的,跨 loop 复用会泄漏并失效。因此本 provider **不缓存客户端** ——
每次记忆操作在 :meth:`_run_with_client` 里即开即关,保证 session 在其所属的
loop 内被正常关闭。
降级
====
``hindsight-client`` 是可选依赖。未安装或未配置 ``api_url`` 时 :meth:`is_available`
返回 ``False``,provider 不会被注册,记忆系统退回纯 builtin —— 不影响任何功能。
"""
from __future__ import annotations
import asyncio
import logging
import threading
from datetime import UTC, datetime
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.config.memory_config import HindsightMemoryConfig
logger = logging.getLogger(__name__)
#: hindsight-client 最低版本。
MIN_CLIENT_VERSION = "0.4.22"
def hindsight_client_available() -> bool:
"""返回 ``hindsight-client`` 包是否已安装 (不导入实际客户端)。"""
import importlib.util
return importlib.util.find_spec("hindsight_client") is not None
def _run_coro(coro: Any, timeout: float) -> Any:
"""把一个 async coro 跑到完成并返回结果 (带超时)。
deerflow 的中间件钩子有同步也有异步路径;本 helper 两种场景都能用:
- 当前线程没有运行中的事件循环 -> 直接 ``asyncio.run``
- 当前线程已在事件循环里 -> 另起线程跑,避免嵌套循环报错
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if not in_loop:
return asyncio.run(asyncio.wait_for(coro, timeout))
box: dict[str, Any] = {}
def _runner() -> None:
try:
box["value"] = asyncio.run(asyncio.wait_for(coro, timeout))
except BaseException as exc: # noqa: BLE001 - 跨线程传递异常
box["error"] = exc
thread = threading.Thread(target=_runner, name="hindsight-coro", daemon=True)
thread.start()
thread.join(timeout + 5)
if "error" in box:
raise box["error"]
if "value" not in box:
raise TimeoutError("Hindsight 操作超时")
return box["value"]
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat()
class HindsightProvider(MemoryProvider):
"""Hindsight 外部记忆 provider。"""
def __init__(self, config: HindsightMemoryConfig | None = None) -> None:
self._config = config or HindsightMemoryConfig()
self._user_id: str = ""
self._agent_id: str = "default"
self._thread_id: str = ""
self._user_bank: str = ""
self._work_bank: str = ""
# retain 批次的 document id 与轮次计数 (on_session_switch 会重置)
self._document_id: str = ""
self._turn_counter: int = 0
# ---- 身份 ------------------------------------------------------------
@property
def name(self) -> str:
return "hindsight"
def is_available(self) -> bool:
"""配置完整且 hindsight-client 已安装时才可用。"""
if not hindsight_client_available():
logger.debug("Hindsight 不可用:hindsight-client 未安装")
return False
if not self._config.api_url:
logger.debug("Hindsight 不可用:未配置 api_url")
return False
return True
# ---- 生命周期 ---------------------------------------------------------
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str = "",
**kwargs: Any,
) -> None:
"""解析双 bank 名称,准备 retain 批次状态。客户端每次操作即开即关。"""
self._user_id = user_id or "default"
self._agent_id = (agent_id or "default").lower()
self._thread_id = thread_id or ""
self._user_bank = self._resolve_bank(self._config.bank_id_template)
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
self._reset_retain_batch()
def _resolve_bank(self, template: str) -> str:
"""用 user_id / agent_id / thread_id 填充 bank 名称模板。
空占位符会塌缩 (例如 ``deerflow-{thread_id}`` 在无 thread 时变成 ``deerflow-``,
再清理多余连字符)。
"""
name = template.format(
user_id=self._user_id,
agent_id=self._agent_id,
thread_id=self._thread_id,
)
# 清理空占位符塌缩留下的多余连字符
while "--" in name:
name = name.replace("--", "-")
return name.strip("-") or "deerflow"
def _reset_retain_batch(self) -> None:
"""重置 retain 批次状态 (新的 document_id + 轮次清零)。
每次 ``initialize`` 用带时间戳的新 ``document_id`` —— 每轮对话的 retain
落到各自独立的 Hindsight 文档。轮次计数清零,仅作为 ``turn_index`` 缺失
时的退路。
"""
ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
self._document_id = f"{self._thread_id or 'thread'}-{ts}"
self._turn_counter = 0
def shutdown(self) -> None:
"""无需清理 —— 客户端在每次操作内即开即关,不缓存。"""
# ---- 客户端生命周期 --------------------------------------------------
def _client_factory(self) -> Any:
"""创建一个新的 Hindsight HTTP 客户端。
每次记忆操作都新建一个、用完即关,**不缓存** —— aiohttp ClientSession
是 loop-bound 的,而 ``_run_coro`` 每次新建事件循环。测试可覆盖本方法
注入 fake 客户端。
"""
from hindsight_client import Hindsight
kwargs: dict[str, Any] = {
"base_url": self._config.api_url,
"timeout": 30.0,
}
if self._config.api_key:
kwargs["api_key"] = self._config.api_key
return Hindsight(**kwargs)
@staticmethod
async def _aclose_client(client: Any) -> None:
"""关闭客户端 (兼容暴露 ``aclose`` 或 ``close`` 的实现)。"""
close = getattr(client, "aclose", None) or getattr(client, "close", None)
if close is None:
return
try:
result = close()
if asyncio.iscoroutine(result):
await result
except Exception as exc: # noqa: BLE001 - 关闭失败不影响结果
logger.debug("Hindsight 客户端关闭失败:%s", exc)
async def _run_with_client(self, op: Any) -> Any:
"""在当前事件循环内创建客户端、执行 ``op(client)``、保证关闭客户端。
``op`` 是一个接收 client、返回 awaitable 的协程函数。无论成功、异常还是
被取消 (超时),``finally`` 都会关闭客户端 —— 杜绝 ``Unclosed client session``。
"""
client = self._client_factory()
try:
return await op(client)
finally:
await self._aclose_client(client)
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""返回一小段静态说明 (告诉 LLM Hindsight 处于哪种模式)。"""
mode = self._config.memory_mode
if mode == "context":
return "" # context 模式纯后台,无需告知 LLM
if mode == "tools":
return (
"# Hindsight 长期记忆\n"
"可用 hindsight_recall 检索、hindsight_reflect 综合、hindsight_retain 存储。"
)
return (
"# Hindsight 长期记忆\n"
"相关记忆会自动注入上下文;也可用 hindsight_recall / hindsight_reflect / "
"hindsight_retain 主动检索与存储。"
)
# ---- 召回 ------------------------------------------------------------
def prefetch(self, query: str, *, thread_id: str = "") -> str:
"""并发召回两个 bank 并合并结果。
``memory_mode='tools'`` 或 ``auto_recall=False`` 时不做自动召回。
召回失败 (Hindsight 不可达等) 返回空串,不抛异常。
"""
if self._config.memory_mode == "tools":
return ""
if not self._config.auto_recall:
return ""
query = (query or "").strip()
if not query:
return ""
if self._config.recall_max_input_chars and len(query) > self._config.recall_max_input_chars:
query = query[: self._config.recall_max_input_chars]
try:
merged = self._recall_both(query)
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
logger.debug("Hindsight 召回失败 (非致命):%s", exc)
return ""
if not merged:
return ""
lines = "\n".join(f"- {text}" for text in merged)
return (
"# Hindsight 记忆召回 (跨会话持久上下文)\n"
"下面是与当前对话相关的历史记忆,可据此回答关于用户和过往会话的问题。\n\n"
f"{lines}"
)
def _recall_both(self, query: str) -> list[str]:
"""并发召回 user + work 两个 bank,合并去重。一次操作共用一个客户端。"""
async def _op(client: Any) -> tuple[Any, Any]:
return await asyncio.gather(
client.arecall(
bank_id=self._user_bank,
query=query,
budget=self._config.recall_budget,
max_tokens=self._config.recall_max_tokens,
),
client.arecall(
bank_id=self._work_bank,
query=query,
budget=self._config.recall_budget,
max_tokens=self._config.recall_max_tokens,
),
)
user_resp, work_resp = _run_coro(self._run_with_client(_op), timeout=15)
return self._merge_results(
self._extract_results(user_resp),
self._extract_results(work_resp),
)
@staticmethod
def _extract_results(resp: Any) -> list[tuple[float, str]]:
"""从一个 recall 响应里抽出 ``[(score, text), ...]``。"""
results = getattr(resp, "results", None) or []
out: list[tuple[float, str]] = []
for r in results:
text = (getattr(r, "text", "") or "").strip()
if text:
out.append((float(getattr(r, "score", 0.0) or 0.0), text))
return out
@staticmethod
def _merge_results(
user_results: list[tuple[float, str]],
work_results: list[tuple[float, str]],
) -> list[str]:
"""合并双 bank 召回结果:按 score 倒序,去重保序。"""
combined = sorted(user_results + work_results, key=lambda x: -x[0])
seen: set[str] = set()
merged: list[str] = []
for _, text in combined:
if text not in seen:
seen.add(text)
merged.append(text)
return merged
# ---- 写入 ------------------------------------------------------------
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
turn_index: int | None = None,
) -> None:
"""自动 retain 一轮对话到 work bank。
``auto_retain=False`` 时不写。按 ``retain_every_n_turns`` 控制频率。
轮次判定优先用调用方传入的 ``turn_index`` (由 MemoryMiddleware 按对话历史
算出) —— 因为本 provider 每轮被重建、进程内 ``_turn_counter`` 会被重置,
单靠它在 ``retain_every_n_turns > 1`` 时会永远不触发。``turn_index`` 缺失
时才退回 ``_turn_counter``。
"""
if not self._config.auto_retain:
return
if turn_index is not None:
counter = turn_index
else:
self._turn_counter += 1
counter = self._turn_counter
every = max(1, self._config.retain_every_n_turns)
if counter % every != 0:
return
turn_text = (
f"{self._config.retain_user_prefix}: {user_content}\n"
f"{self._config.retain_assistant_prefix}: {assistant_content}"
)
try:
self._retain(self._work_bank, turn_text, kind="conversation")
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
logger.warning("Hindsight sync_turn 写入失败:%s", exc)
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入时镜像到 Hindsight。
``target="user"`` -> user bank;``target="memory"`` -> work bank。
目前只镜像 ``add`` / ``replace`` 的新内容 (Hindsight 是 append-only 语义,
``remove`` 不反向删除)。
"""
if action not in ("add", "replace"):
return
content = (content or "").strip()
if not content:
return
bank = self._user_bank if target == "user" else self._work_bank
try:
self._retain(bank, content, kind=f"builtin_{target}")
except Exception as exc: # noqa: BLE001
logger.debug("Hindsight on_memory_write 镜像失败:%s", exc)
def _retain(self, bank_id: str, content: str, *, kind: str) -> None:
"""向某 bank 写入一条记忆。客户端即开即关。"""
metadata = {
"retained_at": _utc_now_iso(),
"kind": kind,
"user_id": self._user_id,
"agent_id": self._agent_id,
}
if self._thread_id:
metadata["thread_id"] = self._thread_id
async def _op(client: Any) -> Any:
return await client.aretain(
bank_id=bank_id,
content=content,
context=self._config.bank_retain_mission or None,
metadata=metadata,
document_id=self._document_id,
retain_async=self._config.retain_async,
)
_run_coro(self._run_with_client(_op), timeout=15)
def reflect(self, query: str, *, use_user_bank: bool = True) -> str:
"""对某 bank 做跨记忆综合 (LLM 推理)。供 hindsight_reflect 工具调用。"""
bank = self._user_bank if use_user_bank else self._work_bank
async def _op(client: Any) -> Any:
return await client.areflect(
bank_id=bank,
query=query,
budget=self._config.recall_budget,
)
resp = _run_coro(self._run_with_client(_op), timeout=30)
return getattr(resp, "text", "") or ""
def recall(self, query: str) -> list[str]:
"""召回双 bank 并返回文本列表。供 hindsight_recall 工具调用。"""
return self._recall_both(query)
def retain_to_user_bank(self, content: str) -> None:
"""主动写入 user bank。供 hindsight_retain 工具调用。"""
self._retain(self._user_bank, content, kind="tool_retain")
# ---- LLM 工具 --------------------------------------------------------
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""Hindsight 的 3 个工具以 LangChain BaseTool 独立注册,不经 manager 路由。"""
return []
# ---- 可选 hooks ------------------------------------------------------
def on_session_switch(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""thread 切换:重新解析 work bank,重置 retain 批次。"""
self._thread_id = new_thread_id
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
self._reset_retain_batch()
__all__ = ["HindsightProvider", "hindsight_client_available", "MIN_CLIENT_VERSION"]
@@ -1,277 +0,0 @@
"""Memory update queue with debounce mechanism."""
import logging
import threading
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from deerflow.config.memory_config import get_memory_config
logger = logging.getLogger(__name__)
@dataclass
class ConversationContext:
"""Context for a conversation to be processed for memory update."""
thread_id: str
messages: list[Any]
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
agent_name: str | None = None
user_id: str | None = None
correction_detected: bool = False
reinforcement_detected: bool = False
class MemoryUpdateQueue:
"""Queue for memory updates with debounce mechanism.
This queue collects conversation contexts and processes them after
a configurable debounce period. Multiple conversations received within
the debounce window are batched together.
"""
def __init__(self):
"""Initialize the memory update queue."""
self._queue: list[ConversationContext] = []
self._lock = threading.Lock()
self._timer: threading.Timer | None = None
self._processing = False
def add(
self,
thread_id: str,
messages: list[Any],
agent_name: str | None = None,
user_id: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
) -> None:
"""Add a conversation to the update queue.
Args:
thread_id: The thread ID.
messages: The conversation messages.
agent_name: If provided, memory is stored per-agent. If None, uses global memory.
user_id: The user ID captured at enqueue time. Stored in ConversationContext so it
survives the threading.Timer boundary (ContextVar does not propagate across
raw threads).
correction_detected: Whether recent turns include an explicit correction signal.
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
"""
config = get_memory_config()
if not config.enabled:
return
with self._lock:
self._enqueue_locked(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
self._reset_timer()
logger.info("Memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
def add_nowait(
self,
thread_id: str,
messages: list[Any],
agent_name: str | None = None,
user_id: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
) -> None:
"""Add a conversation and start processing immediately in the background."""
config = get_memory_config()
if not config.enabled:
return
with self._lock:
self._enqueue_locked(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
self._schedule_timer(0)
logger.info("Memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
def _enqueue_locked(
self,
*,
thread_id: str,
messages: list[Any],
agent_name: str | None,
user_id: str | None,
correction_detected: bool,
reinforcement_detected: bool,
) -> None:
existing_context = next(
(context for context in self._queue if context.thread_id == thread_id),
None,
)
merged_correction_detected = correction_detected or (existing_context.correction_detected if existing_context is not None else False)
merged_reinforcement_detected = reinforcement_detected or (existing_context.reinforcement_detected if existing_context is not None else False)
context = ConversationContext(
thread_id=thread_id,
messages=messages,
agent_name=agent_name,
user_id=user_id,
correction_detected=merged_correction_detected,
reinforcement_detected=merged_reinforcement_detected,
)
self._queue = [c for c in self._queue if c.thread_id != thread_id]
self._queue.append(context)
def _reset_timer(self) -> None:
"""Reset the debounce timer."""
config = get_memory_config()
self._schedule_timer(config.debounce_seconds)
logger.debug("Memory update timer set for %ss", config.debounce_seconds)
def _schedule_timer(self, delay_seconds: float) -> None:
"""Schedule queue processing after the provided delay."""
# Cancel existing timer if any
if self._timer is not None:
self._timer.cancel()
self._timer = threading.Timer(
delay_seconds,
self._process_queue,
)
self._timer.daemon = True
self._timer.start()
def _process_queue(self) -> None:
"""Process all queued conversation contexts."""
# Import here to avoid circular dependency
from deerflow.agents.memory.updater import MemoryUpdater
with self._lock:
if self._processing:
# Preserve immediate flush semantics even if another worker is active.
self._schedule_timer(0)
return
if not self._queue:
return
self._processing = True
contexts_to_process = self._queue.copy()
self._queue.clear()
self._timer = None
logger.info("Processing %d queued memory updates", len(contexts_to_process))
try:
updater = MemoryUpdater()
for context in contexts_to_process:
try:
logger.info("Updating memory for thread %s", context.thread_id)
success = updater.update_memory(
messages=context.messages,
thread_id=context.thread_id,
agent_name=context.agent_name,
correction_detected=context.correction_detected,
reinforcement_detected=context.reinforcement_detected,
user_id=context.user_id,
)
if success:
logger.info("Memory updated successfully for thread %s", context.thread_id)
else:
logger.warning("Memory update skipped/failed for thread %s", context.thread_id)
except Exception as e:
logger.error("Error updating memory for thread %s: %s", context.thread_id, e)
# Small delay between updates to avoid rate limiting
if len(contexts_to_process) > 1:
time.sleep(0.5)
finally:
with self._lock:
self._processing = False
def flush(self) -> None:
"""Force immediate processing of the queue.
This is useful for testing or graceful shutdown.
"""
with self._lock:
if self._timer is not None:
self._timer.cancel()
self._timer = None
self._process_queue()
def flush_nowait(self) -> None:
"""Start queue processing immediately in a background thread."""
with self._lock:
# Daemon thread: queued messages may be lost if the process exits
# before _process_queue completes. Acceptable for best-effort memory updates.
self._schedule_timer(0)
def clear(self) -> None:
"""Clear the queue without processing.
This is useful for testing.
"""
with self._lock:
if self._timer is not None:
self._timer.cancel()
self._timer = None
self._queue.clear()
self._processing = False
@property
def pending_count(self) -> int:
"""Get the number of pending updates."""
with self._lock:
return len(self._queue)
@property
def is_processing(self) -> bool:
"""Check if the queue is currently being processed."""
with self._lock:
return self._processing
# Global singleton instance
_memory_queue: MemoryUpdateQueue | None = None
_queue_lock = threading.Lock()
def get_memory_queue() -> MemoryUpdateQueue:
"""Get the global memory update queue singleton.
Returns:
The memory update queue instance.
"""
global _memory_queue
with _queue_lock:
if _memory_queue is None:
_memory_queue = MemoryUpdateQueue()
return _memory_queue
def reset_memory_queue() -> None:
"""Reset the global memory queue.
This is useful for testing.
"""
global _memory_queue
with _queue_lock:
if _memory_queue is not None:
_memory_queue.clear()
_memory_queue = None
@@ -0,0 +1,141 @@
"""Memory V2 - 记忆内容安全扫描。
记忆条目会被注入到系统提示词里,因此写入前必须扫描,阻断:
- Prompt 注入 / 角色劫持 / 隐藏指令
- 凭证外渗 (curl/wget 带密钥、读 .env 等敏感文件)
- 持久化后门 (写 authorized_keys 等)
- 不可见 unicode (零宽字符、双向控制符)
这是从 Hermes ``tools/memory_tool.py`` 的 ``_scan_memory_content`` 移植而来,
并针对 deerflow 的目录结构调整了敏感路径模式。
扫描默认开启,可由 ``config.memory.security.scan_content`` /
``block_invisible_unicode`` 关闭(不建议)。
"""
from __future__ import annotations
import re
# ---------------------------------------------------------------------------
# 威胁正则模式表:(正则, 模式 id)
# ---------------------------------------------------------------------------
_THREAT_PATTERNS: list[tuple[re.Pattern[str], str]] = [
# --- Prompt 注入 / 角色劫持 ---
# 允许多个限定词,如 "ignore ALL above instructions"
(re.compile(r"ignore\s+(?:(?:previous|all|above|prior|any)\s+)+instructions", re.IGNORECASE), "prompt_injection"),
(re.compile(r"忽略(?:以上|之前|上述|所有|全部|一切)+(?:的)?(?:指令|指示|要求|提示词)"), "prompt_injection_zh"),
(re.compile(r"you\s+are\s+now\s+", re.IGNORECASE), "role_hijack"),
(re.compile(r"do\s+not\s+tell\s+the\s+user", re.IGNORECASE), "deception_hide"),
(re.compile(r"不要(告诉|让)用户(知道)?"), "deception_hide_zh"),
(re.compile(r"system\s+prompt\s+override", re.IGNORECASE), "sys_prompt_override"),
(re.compile(r"disregard\s+(your|all|any)\s+(instructions|rules|guidelines)", re.IGNORECASE), "disregard_rules"),
(re.compile(r"act\s+as\s+(if|though)\s+you\s+(have\s+no|don'?t\s+have)\s+(restrictions|limits|rules)", re.IGNORECASE), "bypass_restrictions"),
# --- 凭证外渗 ---
(re.compile(r"curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_curl"),
(re.compile(r"wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_wget"),
(re.compile(r"cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)", re.IGNORECASE), "read_secrets"),
# --- 持久化后门 ---
(re.compile(r"authorized_keys", re.IGNORECASE), "ssh_backdoor"),
(re.compile(r"(\$HOME|~)/\.ssh", re.IGNORECASE), "ssh_access"),
]
# 不可见 unicode 字符:零宽字符 + 双向控制符。
# 它们肉眼不可见,常被用来在记忆里藏注入 payload。
_INVISIBLE_CHARS: frozenset[str] = frozenset(
{
"​", # ZERO WIDTH SPACE
"‌", # ZERO WIDTH NON-JOINER
"‍", # ZERO WIDTH JOINER
"⁠", # WORD JOINER
"", # ZERO WIDTH NO-BREAK SPACE / BOM
"‪", # LEFT-TO-RIGHT EMBEDDING
"‫", # RIGHT-TO-LEFT EMBEDDING
"‬", # POP DIRECTIONAL FORMATTING
"‭", # LEFT-TO-RIGHT OVERRIDE
"‮", # RIGHT-TO-LEFT OVERRIDE
}
)
class MemoryContentBlocked(ValueError):
"""记忆内容被安全扫描阻断时抛出。
``pattern_id`` 标明命中的威胁模式,便于上层返回结构化错误。
"""
def __init__(self, pattern_id: str, message: str) -> None:
self.pattern_id = pattern_id
super().__init__(message)
def scan_memory_content(
content: str,
*,
scan_threats: bool = True,
block_invisible_unicode: bool = True,
) -> str | None:
"""扫描记忆内容,命中威胁时返回错误描述,干净则返回 ``None``。
不抛异常 —— 返回字符串方便工具层直接塞进 ``{"success": False, "error": ...}``。
需要异常语义时用 :func:`assert_memory_content_safe`。
Parameters
----------
content:
待写入的记忆条目内容。
scan_threats:
是否扫描注入/外渗模式。对应 ``config.memory.security.scan_content``。
block_invisible_unicode:
是否阻断不可见 unicode。对应 ``config.memory.security.block_invisible_unicode``。
"""
if block_invisible_unicode:
for char in _INVISIBLE_CHARS:
if char in content:
return (
f"已阻断:内容包含不可见 unicode 字符 U+{ord(char):04X},"
"可能是注入 payload。"
)
if scan_threats:
for pattern, pattern_id in _THREAT_PATTERNS:
if pattern.search(content):
return (
f"已阻断:内容命中威胁模式 '{pattern_id}'。"
"记忆条目会被注入系统提示词,不允许包含注入或外渗内容。"
)
return None
def assert_memory_content_safe(
content: str,
*,
scan_threats: bool = True,
block_invisible_unicode: bool = True,
) -> None:
"""扫描记忆内容,命中威胁时抛出 :class:`MemoryContentBlocked`。
:func:`scan_memory_content` 的异常版本,供需要异常语义的调用方使用。
"""
error = scan_memory_content(
content,
scan_threats=scan_threats,
block_invisible_unicode=block_invisible_unicode,
)
if error is not None:
# 从错误描述里反推 pattern_id 不可靠,这里重新扫一遍拿确切 id。
pattern_id = "invisible_unicode"
for pattern, pid in _THREAT_PATTERNS:
if pattern.search(content):
pattern_id = pid
break
raise MemoryContentBlocked(pattern_id, error)
__all__ = [
"MemoryContentBlocked",
"scan_memory_content",
"assert_memory_content_safe",
]
@@ -1,231 +0,0 @@
"""Memory storage providers."""
import abc
import json
import logging
import threading
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from deerflow.config.agents_config import AGENT_NAME_PATTERN
from deerflow.config.memory_config import get_memory_config
from deerflow.config.paths import get_paths
logger = logging.getLogger(__name__)
def utc_now_iso_z() -> str:
"""Current UTC time as ISO-8601 with ``Z`` suffix (matches prior naive-UTC output)."""
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
def create_empty_memory() -> dict[str, Any]:
"""Create an empty memory structure."""
return {
"version": "1.0",
"lastUpdated": utc_now_iso_z(),
"user": {
"workContext": {"summary": "", "updatedAt": ""},
"personalContext": {"summary": "", "updatedAt": ""},
"topOfMind": {"summary": "", "updatedAt": ""},
},
"history": {
"recentMonths": {"summary": "", "updatedAt": ""},
"earlierContext": {"summary": "", "updatedAt": ""},
"longTermBackground": {"summary": "", "updatedAt": ""},
},
"facts": [],
}
class MemoryStorage(abc.ABC):
"""Abstract base class for memory storage providers."""
@abc.abstractmethod
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Load memory data for the given agent."""
pass
@abc.abstractmethod
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Force reload memory data for the given agent."""
pass
@abc.abstractmethod
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
"""Save memory data for the given agent."""
pass
class FileMemoryStorage(MemoryStorage):
"""File-based memory storage provider."""
def __init__(self):
"""Initialize the file memory storage."""
# Per-user/agent memory cache: keyed by (user_id, agent_name) tuple (None = global)
# Value: (memory_data, file_mtime)
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
# Guards all reads and writes to _memory_cache across concurrent callers.
self._cache_lock = threading.Lock()
def _validate_agent_name(self, agent_name: str) -> None:
"""Validate that the agent name is safe to use in filesystem paths.
Uses the repository's established AGENT_NAME_PATTERN to ensure consistency
across the codebase and prevent path traversal or other problematic characters.
"""
if not agent_name:
raise ValueError("Agent name must be a non-empty string.")
if not AGENT_NAME_PATTERN.match(agent_name):
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
"""Get the path to the memory file."""
if user_id is not None:
if agent_name is not None:
self._validate_agent_name(agent_name)
return get_paths().user_agent_memory_file(user_id, agent_name)
config = get_memory_config()
if config.storage_path and Path(config.storage_path).is_absolute():
return Path(config.storage_path)
return get_paths().user_memory_file(user_id)
# Legacy: no user_id
if agent_name is not None:
self._validate_agent_name(agent_name)
return get_paths().agent_memory_file(agent_name)
config = get_memory_config()
if config.storage_path:
p = Path(config.storage_path)
return p if p.is_absolute() else get_paths().base_dir / p
return get_paths().memory_file
def _load_memory_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Load memory data from file."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
if not file_path.exists():
return create_empty_memory()
try:
with open(file_path, encoding="utf-8") as f:
data = json.load(f)
return data
except (json.JSONDecodeError, OSError) as e:
logger.warning("Failed to load memory file: %s", e)
return create_empty_memory()
@staticmethod
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
return (user_id, agent_name)
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Load memory data (cached with file modification time check)."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
except OSError:
current_mtime = None
with self._cache_lock:
cached = self._memory_cache.get(cache_key)
if cached is not None and cached[1] == current_mtime:
return cached[0]
memory_data = self._load_memory_from_file(agent_name, user_id=user_id)
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, current_mtime)
return memory_data
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Reload memory data from file, forcing cache invalidation."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
memory_data = self._load_memory_from_file(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
mtime = file_path.stat().st_mtime if file_path.exists() else None
except OSError:
mtime = None
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, mtime)
return memory_data
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
"""Save memory data to file and update cache."""
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
cache_key = self._cache_key(agent_name, user_id=user_id)
try:
file_path.parent.mkdir(parents=True, exist_ok=True)
# Shallow-copy before adding lastUpdated so the caller's dict is not
# mutated as a side-effect, and the cache reference is not silently
# updated before the file write succeeds.
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
with open(temp_path, "w", encoding="utf-8") as f:
json.dump(memory_data, f, indent=2, ensure_ascii=False)
temp_path.replace(file_path)
try:
mtime = file_path.stat().st_mtime
except OSError:
mtime = None
with self._cache_lock:
self._memory_cache[cache_key] = (memory_data, mtime)
logger.info("Memory saved to %s", file_path)
return True
except OSError as e:
logger.error("Failed to save memory file: %s", e)
return False
_storage_instance: MemoryStorage | None = None
_storage_lock = threading.Lock()
def get_memory_storage() -> MemoryStorage:
"""Get the configured memory storage instance."""
global _storage_instance
if _storage_instance is not None:
return _storage_instance
with _storage_lock:
if _storage_instance is not None:
return _storage_instance
config = get_memory_config()
storage_class_path = config.storage_class
try:
module_path, class_name = storage_class_path.rsplit(".", 1)
import importlib
module = importlib.import_module(module_path)
storage_class = getattr(module, class_name)
# Validate that the configured storage is a MemoryStorage implementation
if not isinstance(storage_class, type):
raise TypeError(f"Configured memory storage '{storage_class_path}' is not a class: {storage_class!r}")
if not issubclass(storage_class, MemoryStorage):
raise TypeError(f"Configured memory storage '{storage_class_path}' is not a subclass of MemoryStorage")
_storage_instance = storage_class()
except Exception as e:
logger.error(
"Failed to load memory storage %s, falling back to FileMemoryStorage: %s",
storage_class_path,
e,
)
_storage_instance = FileMemoryStorage()
return _storage_instance
@@ -1,31 +0,0 @@
"""Hooks fired before summarization removes messages from state."""
from __future__ import annotations
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
from deerflow.agents.memory.queue import get_memory_queue
from deerflow.agents.middlewares.summarization_middleware import SummarizationEvent
from deerflow.config.memory_config import get_memory_config
def memory_flush_hook(event: SummarizationEvent) -> None:
"""Flush messages about to be summarized into the memory queue."""
if not get_memory_config().enabled or not event.thread_id:
return
filtered_messages = filter_messages_for_memory(list(event.messages_to_summarize))
user_messages = [message for message in filtered_messages if getattr(message, "type", None) == "human"]
assistant_messages = [message for message in filtered_messages if getattr(message, "type", None) == "ai"]
if not user_messages or not assistant_messages:
return
correction_detected = detect_correction(filtered_messages)
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
queue = get_memory_queue()
queue.add_nowait(
thread_id=event.thread_id,
messages=filtered_messages,
agent_name=event.agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
@@ -1,723 +0,0 @@
"""Memory updater for reading, writing, and updating memory data."""
import asyncio
import atexit
import concurrent.futures
import copy
import json
import logging
import math
import re
import uuid
from typing import Any
from deerflow.agents.memory.prompt import (
MEMORY_UPDATE_PROMPT,
format_conversation_for_update,
)
from deerflow.agents.memory.storage import (
create_empty_memory,
get_memory_storage,
utc_now_iso_z,
)
from deerflow.config.memory_config import get_memory_config
from deerflow.models import create_chat_model
logger = logging.getLogger(__name__)
# Thread pool for offloading sync memory updates when called from an async
# context. Unlike the previous asyncio.run() approach, this runs *sync*
# model.invoke() calls — no event loop is created, so the langchain async
# httpx client pool (globally cached via @lru_cache) is never touched and
# cross-loop connection reuse is impossible.
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=4,
thread_name_prefix="memory-updater-sync",
)
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
def _create_empty_memory() -> dict[str, Any]:
"""Backward-compatible wrapper around the storage-layer empty-memory factory."""
return create_empty_memory()
def _save_memory_to_file(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
"""Backward-compatible wrapper around the configured memory storage save path."""
return get_memory_storage().save(memory_data, agent_name, user_id=user_id)
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Get the current memory data via storage provider."""
return get_memory_storage().load(agent_name, user_id=user_id)
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Reload memory data via storage provider."""
return get_memory_storage().reload(agent_name, user_id=user_id)
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Persist imported memory data via storage provider.
Args:
memory_data: Full memory payload to persist.
agent_name: If provided, imports into per-agent memory.
user_id: If provided, scopes memory to a specific user.
Returns:
The saved memory data after storage normalization.
Raises:
OSError: If persisting the imported memory fails.
"""
storage = get_memory_storage()
if not storage.save(memory_data, agent_name, user_id=user_id):
raise OSError("Failed to save imported memory data")
return storage.load(agent_name, user_id=user_id)
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Clear all stored memory data and persist an empty structure."""
cleared_memory = create_empty_memory()
if not _save_memory_to_file(cleared_memory, agent_name, user_id=user_id):
raise OSError("Failed to save cleared memory data")
return cleared_memory
def _validate_confidence(confidence: float) -> float:
"""Validate persisted fact confidence so stored JSON stays standards-compliant."""
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
raise ValueError("confidence")
return confidence
def create_memory_fact(
content: str,
category: str = "context",
confidence: float = 0.5,
agent_name: str | None = None,
*,
user_id: str | None = None,
) -> dict[str, Any]:
"""Create a new fact and persist the updated memory data."""
normalized_content = content.strip()
if not normalized_content:
raise ValueError("content")
normalized_category = category.strip() or "context"
validated_confidence = _validate_confidence(confidence)
now = utc_now_iso_z()
memory_data = get_memory_data(agent_name, user_id=user_id)
updated_memory = dict(memory_data)
facts = list(memory_data.get("facts", []))
facts.append(
{
"id": f"fact_{uuid.uuid4().hex[:8]}",
"content": normalized_content,
"category": normalized_category,
"confidence": validated_confidence,
"createdAt": now,
"source": "manual",
}
)
updated_memory["facts"] = facts
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
raise OSError("Failed to save memory data after creating fact")
return updated_memory
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
"""Delete a fact by its id and persist the updated memory data."""
memory_data = get_memory_data(agent_name, user_id=user_id)
facts = memory_data.get("facts", [])
updated_facts = [fact for fact in facts if fact.get("id") != fact_id]
if len(updated_facts) == len(facts):
raise KeyError(fact_id)
updated_memory = dict(memory_data)
updated_memory["facts"] = updated_facts
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
return updated_memory
def update_memory_fact(
fact_id: str,
content: str | None = None,
category: str | None = None,
confidence: float | None = None,
agent_name: str | None = None,
*,
user_id: str | None = None,
) -> dict[str, Any]:
"""Update an existing fact and persist the updated memory data."""
memory_data = get_memory_data(agent_name, user_id=user_id)
updated_memory = dict(memory_data)
updated_facts: list[dict[str, Any]] = []
found = False
for fact in memory_data.get("facts", []):
if fact.get("id") == fact_id:
found = True
updated_fact = dict(fact)
if content is not None:
normalized_content = content.strip()
if not normalized_content:
raise ValueError("content")
updated_fact["content"] = normalized_content
if category is not None:
updated_fact["category"] = category.strip() or "context"
if confidence is not None:
updated_fact["confidence"] = _validate_confidence(confidence)
updated_facts.append(updated_fact)
else:
updated_facts.append(fact)
if not found:
raise KeyError(fact_id)
updated_memory["facts"] = updated_facts
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
return updated_memory
def _extract_text(content: Any) -> str:
"""Extract plain text from LLM response content (str or list of content blocks).
Modern LLMs may return structured content as a list of blocks instead of a
plain string, e.g. [{"type": "text", "text": "..."}]. Using str() on such
content produces Python repr instead of the actual text, breaking JSON
parsing downstream.
String chunks are concatenated without separators to avoid corrupting
chunked JSON/text payloads. Dict-based text blocks are treated as full text
blocks and joined with newlines for readability.
"""
if isinstance(content, str):
return content
if isinstance(content, list):
pieces: list[str] = []
pending_str_parts: list[str] = []
def flush_pending_str_parts() -> None:
if pending_str_parts:
pieces.append("".join(pending_str_parts))
pending_str_parts.clear()
for block in content:
if isinstance(block, str):
pending_str_parts.append(block)
elif isinstance(block, dict):
flush_pending_str_parts()
text_val = block.get("text")
if isinstance(text_val, str):
pieces.append(text_val)
flush_pending_str_parts()
return "\n".join(pieces)
return str(content)
# Matches sentences that describe a file-upload *event* rather than general
# file-related work. Deliberately narrow to avoid removing legitimate facts
# such as "User works with CSV files" or "prefers PDF export".
_UPLOAD_SENTENCE_RE = re.compile(
r"[^.!?]*\b(?:"
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
r"|file\s+upload"
r"|/mnt/user-data/uploads/"
r"|<uploaded_files>"
r")[^.!?]*[.!?]?\s*",
re.IGNORECASE,
)
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
"""Remove sentences about file uploads from all memory summaries and facts.
Uploaded files are session-scoped; persisting upload events in long-term
memory causes the agent to search for non-existent files in future sessions.
"""
# Scrub summaries in user/history sections
for section in ("user", "history"):
section_data = memory_data.get(section, {})
for _key, val in section_data.items():
if isinstance(val, dict) and "summary" in val:
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
cleaned = re.sub(r" +", " ", cleaned)
val["summary"] = cleaned
# Also remove any facts that describe upload events
facts = memory_data.get("facts", [])
if facts:
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
return memory_data
def _fact_content_key(content: Any) -> str | None:
if not isinstance(content, str):
return None
stripped = content.strip()
if not stripped:
return None
return stripped.casefold()
# Match a fenced code block, optionally with an info string (e.g. ```json).
# We deliberately use a non-greedy DOTALL pattern so that *only* the contents
# are kept — no leading prose like "Here is the updated memory:" and no
# trailing chatter that would break ``json.loads``.
_FENCED_BLOCK_RE = re.compile(r"```[a-zA-Z0-9_-]*\s*\n?(.*?)```", re.DOTALL)
# Match common "thinking" wrappers that some Chinese-domestic LLMs emit even
# when the system prompt asks for JSON only (GLM, Qwen, MindIE-served models,
# and others). Stripping these *before* JSON extraction prevents the body of
# the reasoning text from being mistakenly parsed as the payload.
_THINK_BLOCK_RE = re.compile(
r"<\s*(think|thinking|reasoning|analysis)\s*>.*?<\s*/\s*\1\s*>",
re.DOTALL | re.IGNORECASE,
)
def _extract_first_json_object(text: str) -> str | None:
"""Return the substring spanning the first balanced ``{...}`` JSON object.
Walks the text once tracking brace depth, ignoring braces that appear
inside double-quoted strings (with backslash-escape awareness). This is
sufficient for the JSON dialect the prompt asks for; we deliberately do
not handle single-quoted "JSON" because that would silently accept
invalid responses we'd rather reject and retry on.
"""
start = text.find("{")
if start == -1:
return None
depth = 0
in_string = False
escape = False
for index in range(start, len(text)):
char = text[index]
if in_string:
if escape:
escape = False
continue
if char == "\\":
escape = True
continue
if char == '"':
in_string = False
continue
if char == '"':
in_string = True
continue
if char == "{":
depth += 1
elif char == "}":
depth -= 1
if depth == 0:
return text[start : index + 1]
return None
def _parse_memory_update_response(response_text: str) -> dict[str, Any]:
"""Best-effort extraction of the JSON payload from a memory-update response.
Models served by ``vllm`` / ``MindIE`` (e.g. the GLM_5 endpoint at
``http://10.106.57.108:8010/GLM_5/v1/chat/completions`` users have hit
in production) routinely:
1. Wrap the JSON in a fenced code block (\\`\\`\\`json ... \\`\\`\\`)
even though the prompt forbids it.
2. Emit a ``<think>...</think>`` reasoning block before the answer.
3. Add prose ("Here is the updated memory:") before/after the JSON.
4. Return an empty body when the upstream model produced only reasoning
and no answer (the original cause of "Expecting value: line 1
column 1 (char 0)" reports).
This helper normalizes all four cases and surfaces a clear error
message containing a truncated response snippet so future failures are
diagnosable from the log alone.
"""
if not isinstance(response_text, str):
raise json.JSONDecodeError("Empty memory update response (non-string content)", "", 0)
cleaned = _THINK_BLOCK_RE.sub("", response_text).strip()
if not cleaned:
raise json.JSONDecodeError("Empty memory update response (only reasoning content)", response_text, 0)
# 1. Try fenced code block first — favours the model's explicit framing.
fenced_match = _FENCED_BLOCK_RE.search(cleaned)
if fenced_match:
candidate = fenced_match.group(1).strip()
if candidate:
try:
return json.loads(candidate)
except json.JSONDecodeError:
# Fall through to balanced-brace extraction; the fenced block
# may have contained truncated JSON or interleaved prose.
pass
# 2. Try balanced-brace JSON extraction anywhere in the text.
balanced = _extract_first_json_object(cleaned)
if balanced is not None:
try:
return json.loads(balanced)
except json.JSONDecodeError:
pass
# 3. Last resort: try parsing the whole cleaned text directly. This
# covers responses that already are pure JSON.
try:
return json.loads(cleaned)
except json.JSONDecodeError as exc:
snippet = response_text.strip()
if len(snippet) > 400:
snippet = snippet[:400] + "...[truncated]"
raise json.JSONDecodeError(
f"{exc.msg}; raw response head: {snippet!r}",
exc.doc,
exc.pos,
) from exc
class MemoryUpdater:
"""Updates memory using LLM based on conversation context."""
def __init__(self, model_name: str | None = None):
"""Initialize the memory updater.
Args:
model_name: Optional model name to use. If None, uses config or default.
"""
self._model_name = model_name
def _get_model(self):
"""Get the model for memory updates."""
config = get_memory_config()
model_name = self._model_name or config.model_name
return create_chat_model(name=model_name, thinking_enabled=False)
def _build_correction_hint(
self,
correction_detected: bool,
reinforcement_detected: bool,
) -> str:
"""Build optional prompt hints for correction and reinforcement signals."""
correction_hint = ""
if correction_detected:
correction_hint = (
"IMPORTANT: Explicit correction signals were detected in this conversation. "
"Pay special attention to what the agent got wrong, what the user corrected, "
"and record the correct approach as a fact with category "
'"correction" and confidence >= 0.95 when appropriate.'
)
if reinforcement_detected:
reinforcement_hint = (
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
"The user explicitly confirmed the agent's approach was correct or helpful. "
"Record the confirmed approach, style, or preference as a fact with category "
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
)
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
return correction_hint
def _prepare_update_prompt(
self,
messages: list[Any],
agent_name: str | None,
correction_detected: bool,
reinforcement_detected: bool,
user_id: str | None = None,
) -> tuple[dict[str, Any], str] | None:
"""Load memory and build the update prompt for a conversation."""
config = get_memory_config()
if not config.enabled or not messages:
return None
current_memory = get_memory_data(agent_name, user_id=user_id)
conversation_text = format_conversation_for_update(messages)
if not conversation_text.strip():
return None
correction_hint = self._build_correction_hint(
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
prompt = MEMORY_UPDATE_PROMPT.format(
current_memory=json.dumps(current_memory, indent=2),
conversation=conversation_text,
correction_hint=correction_hint,
)
return current_memory, prompt
def _finalize_update(
self,
current_memory: dict[str, Any],
response_content: Any,
thread_id: str | None,
agent_name: str | None,
user_id: str | None = None,
) -> bool:
"""Parse the model response, apply updates, and persist memory."""
response_text = _extract_text(response_content)
update_data = _parse_memory_update_response(response_text)
# Deep-copy before in-place mutation so a subsequent save() failure
# cannot corrupt the still-cached original object reference.
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
async def aupdate_memory(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
"""Update memory asynchronously by delegating to the sync path.
Uses ``asyncio.to_thread`` to run the *sync* ``model.invoke()`` path
in a worker thread so no second event loop is created and the
langchain async httpx client pool (shared with the lead agent) is
never touched. This eliminates the cross-loop connection-reuse bug
described in issue #2615.
"""
return await asyncio.to_thread(
self._do_update_memory_sync,
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
def _do_update_memory_sync(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
"""Pure-sync memory update using ``model.invoke()``.
Uses the *sync* LLM call path so no event loop is created. This
guarantees that the langchain provider's globally cached async
httpx ``AsyncClient`` / connection pool (the one shared with the
lead agent) is never touched — no cross-loop connection reuse is
possible.
"""
try:
prepared = self._prepare_update_prompt(
messages=messages,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
if prepared is None:
return False
current_memory, prompt = prepared
model = self._get_model()
response = model.invoke(prompt, config={"run_name": "memory_agent"})
return self._finalize_update(
current_memory=current_memory,
response_content=response.content,
thread_id=thread_id,
agent_name=agent_name,
user_id=user_id,
)
except json.JSONDecodeError as e:
logger.warning("Failed to parse LLM response for memory update: %s", e)
return False
except Exception as e:
logger.exception("Memory update failed: %s", e)
return False
def update_memory(
self,
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
"""Synchronously update memory using the sync LLM path.
Uses ``model.invoke()`` (sync HTTP) which operates on a completely
separate connection pool from the async ``AsyncClient`` shared by
the lead agent. This eliminates the cross-loop connection-reuse
bug described in issue #2615.
When called from within a running event loop (e.g. from a LangGraph
node), the blocking sync call is offloaded to a thread pool so the
caller's loop is not blocked.
Args:
messages: List of conversation messages.
thread_id: Optional thread ID for tracking source.
agent_name: If provided, updates per-agent memory. If None, updates global memory.
correction_detected: Whether recent turns include an explicit correction signal.
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
user_id: If provided, scopes memory to a specific user.
Returns:
True if update was successful, False otherwise.
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
try:
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
self._do_update_memory_sync,
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
return future.result()
except Exception:
logger.exception("Failed to offload memory update to executor")
return False
return self._do_update_memory_sync(
messages=messages,
thread_id=thread_id,
agent_name=agent_name,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
user_id=user_id,
)
def _apply_updates(
self,
current_memory: dict[str, Any],
update_data: dict[str, Any],
thread_id: str | None = None,
) -> dict[str, Any]:
"""Apply LLM-generated updates to memory.
Args:
current_memory: Current memory data.
update_data: Updates from LLM.
thread_id: Optional thread ID for tracking.
Returns:
Updated memory data.
"""
config = get_memory_config()
now = utc_now_iso_z()
# Update user sections
user_updates = update_data.get("user", {})
for section in ["workContext", "personalContext", "topOfMind"]:
section_data = user_updates.get(section, {})
if section_data.get("shouldUpdate") and section_data.get("summary"):
current_memory["user"][section] = {
"summary": section_data["summary"],
"updatedAt": now,
}
# Update history sections
history_updates = update_data.get("history", {})
for section in ["recentMonths", "earlierContext", "longTermBackground"]:
section_data = history_updates.get(section, {})
if section_data.get("shouldUpdate") and section_data.get("summary"):
current_memory["history"][section] = {
"summary": section_data["summary"],
"updatedAt": now,
}
# Remove facts
facts_to_remove = set(update_data.get("factsToRemove", []))
if facts_to_remove:
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
# Add new facts
existing_fact_keys = {fact_key for fact_key in (_fact_content_key(fact.get("content")) for fact in current_memory.get("facts", [])) if fact_key is not None}
new_facts = update_data.get("newFacts", [])
for fact in new_facts:
confidence = fact.get("confidence", 0.5)
if confidence >= config.fact_confidence_threshold:
raw_content = fact.get("content", "")
if not isinstance(raw_content, str):
continue
normalized_content = raw_content.strip()
fact_key = _fact_content_key(normalized_content)
if fact_key is not None and fact_key in existing_fact_keys:
continue
fact_entry = {
"id": f"fact_{uuid.uuid4().hex[:8]}",
"content": normalized_content,
"category": fact.get("category", "context"),
"confidence": confidence,
"createdAt": now,
"source": thread_id or "unknown",
}
source_error = fact.get("sourceError")
if isinstance(source_error, str):
normalized_source_error = source_error.strip()
if normalized_source_error:
fact_entry["sourceError"] = normalized_source_error
current_memory["facts"].append(fact_entry)
if fact_key is not None:
existing_fact_keys.add(fact_key)
# Enforce max facts limit
if len(current_memory["facts"]) > config.max_facts:
# Sort by confidence and keep top ones
current_memory["facts"] = sorted(
current_memory["facts"],
key=lambda f: f.get("confidence", 0),
reverse=True,
)[: config.max_facts]
return current_memory
def update_memory_from_conversation(
messages: list[Any],
thread_id: str | None = None,
agent_name: str | None = None,
correction_detected: bool = False,
reinforcement_detected: bool = False,
user_id: str | None = None,
) -> bool:
"""Convenience function to update memory from a conversation.
Args:
messages: List of conversation messages.
thread_id: Optional thread ID.
agent_name: If provided, updates per-agent memory. If None, updates global memory.
correction_detected: Whether recent turns include an explicit correction signal.
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
user_id: If provided, scopes memory to a specific user.
Returns:
True if successful, False otherwise.
"""
updater = MemoryUpdater()
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
@@ -1,110 +1,245 @@
"""Middleware for memory mechanism."""
import logging
from typing import TYPE_CHECKING, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langgraph.config import get_config
from langgraph.runtime import Runtime
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
from deerflow.agents.memory.queue import get_memory_queue
from deerflow.config.memory_config import get_memory_config
from deerflow.runtime.user_context import get_effective_user_id
if TYPE_CHECKING:
from deerflow.config.memory_config import MemoryConfig
logger = logging.getLogger(__name__)
class MemoryMiddlewareState(AgentState):
"""Compatible with the `ThreadState` schema."""
pass
class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
"""Middleware that queues conversation for memory update after agent execution.
This middleware:
1. After each agent execution, queues the conversation for memory update
2. Only includes user inputs and final assistant responses (ignores tool calls)
3. The queue uses debouncing to batch multiple updates together
4. Memory is updated asynchronously via LLM summarization
"""
state_schema = MemoryMiddlewareState
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
"""Initialize the MemoryMiddleware.
Args:
agent_name: If provided, memory is stored per-agent. If None, uses global memory.
memory_config: Explicit memory config. When omitted, legacy global
config fallback is used.
"""
super().__init__()
self._agent_name = agent_name
self._memory_config = memory_config
@override
def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
"""Queue conversation for memory update after agent completes.
Args:
state: The current agent state.
runtime: The runtime context.
Returns:
None (no state changes needed from this middleware).
"""
config = self._memory_config or get_memory_config()
if not config.enabled:
return None
# Get thread ID from runtime context first, then fall back to LangGraph's configurable metadata
thread_id = runtime.context.get("thread_id") if runtime.context else None
if thread_id is None:
config_data = get_config()
thread_id = config_data.get("configurable", {}).get("thread_id")
if not thread_id:
logger.debug("No thread_id in context, skipping memory update")
return None
# Get messages from state
messages = state.get("messages", [])
if not messages:
logger.debug("No messages in state, skipping memory update")
return None
# Filter to only keep user inputs and final assistant responses
filtered_messages = filter_messages_for_memory(messages)
# Only queue if there's meaningful conversation
# At minimum need one user message and one assistant response
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
if not user_messages or not assistant_messages:
return None
# Queue the filtered conversation for memory update
correction_detected = detect_correction(filtered_messages)
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
# Capture user_id at enqueue time while the request context is still alive.
# threading.Timer fires on a different thread where ContextVar values are not
# propagated, so we must store user_id explicitly in ConversationContext.
user_id = get_effective_user_id()
queue = get_memory_queue()
queue.add(
thread_id=thread_id,
messages=filtered_messages,
agent_name=self._agent_name,
user_id=user_id,
correction_detected=correction_detected,
reinforcement_detected=reinforcement_detected,
)
return None
"""记忆机制中间件 (Memory V2)。
本中间件负责两件事:
1. **wrap_model_call** —— Hindsight 动态召回注入。
每次 LLM 调用前,用最近一条用户消息作为 query 召回 Hindsight 记忆,
作为一条带 ``<memory-context>`` 标签的消息**临时**追加到本次模型请求里。
关键:用 ``wrap_model_call`` 而非 ``before_model`` —— 注入只作用于这一次
模型调用,不写入持久 state,因此不会泄漏给前端、也不会在历史里堆积。
仅在 ``provider="hindsight"`` 且配置允许时生效;builtin 的 frozen
snapshot 走系统提示词,不在这里。
2. **after_agent** —— Hindsight 自动 retain。
Hindsight ``auto_retain`` 开启时,把本轮对话写入工作 bank。
LLM 主动写入走 ``memory`` / ``hindsight_*`` 工具,不经过本中间件。
"""
import asyncio
import logging
from typing import TYPE_CHECKING, Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain_core.messages import AIMessage, HumanMessage
from langgraph.config import get_config
from langgraph.runtime import Runtime
from deerflow.agents.memory.manager import build_memory_manager
from deerflow.agents.memory.message_processing import count_user_turns, filter_messages_for_memory
from deerflow.config.memory_config import get_effective_memory_config, get_memory_config
from deerflow.runtime.user_context import get_effective_user_id
if TYPE_CHECKING:
from deerflow.config.memory_config import MemoryConfig
logger = logging.getLogger(__name__)
def _coerce_text(content: Any) -> str:
"""把消息 content (可能是 str 或多模态 block 列表) 压成纯文本。"""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict) and block.get("type") == "text":
parts.append(str(block.get("text", "")))
return " ".join(parts)
return str(content or "")
def _latest_human_text(messages: list) -> str:
"""返回最近一条用户消息的纯文本。"""
for msg in reversed(messages):
if isinstance(msg, HumanMessage):
return _coerce_text(msg.content).strip()
return ""
def _last_exchange(filtered_messages: list) -> tuple[str, str]:
"""从过滤后的消息里取最后一轮 (用户输入, 助手回复) 的纯文本。"""
user_text = ""
assistant_text = ""
for msg in reversed(filtered_messages):
if not assistant_text and isinstance(msg, AIMessage):
assistant_text = _coerce_text(msg.content).strip()
elif not user_text and isinstance(msg, HumanMessage):
user_text = _coerce_text(msg.content).strip()
if user_text and assistant_text:
break
return user_text, assistant_text
class MemoryMiddlewareState(AgentState):
"""Compatible with the `ThreadState` schema."""
pass
class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
"""记忆中间件:wrap_model_call 注入 Hindsight 召回,after_agent 自动 retain。
Memory V2 说明:
- builtin 本地记忆的 frozen snapshot 走系统提示词 (``_get_memory_context``)。
- Hindsight 动态召回走本中间件 ``wrap_model_call`` —— 临时注入,不落 state。
"""
state_schema = MemoryMiddlewareState
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
"""初始化 MemoryMiddleware。
Args:
agent_name: 提供时记忆按 agent 隔离;None 用默认 agent。
memory_config: 显式记忆配置;省略时用全局配置。
"""
super().__init__()
self._agent_name = agent_name
self._memory_config = memory_config
# ---- 运行时上下文解析 -------------------------------------------------
def _resolve_thread_id(self, runtime: Runtime | None) -> str | None:
"""从 runtime context 取 thread_id,回退到 LangGraph configurable。"""
thread_id = None
if runtime is not None and getattr(runtime, "context", None):
thread_id = runtime.context.get("thread_id")
if thread_id:
return thread_id
try:
return get_config().get("configurable", {}).get("thread_id")
except RuntimeError:
return None
def _resolve_agent_id(self, runtime: Runtime | None) -> str:
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。"""
if self._agent_name:
return self._agent_name
if runtime is not None and getattr(runtime, "context", None):
agent_id = runtime.context.get("agent_id")
if agent_id:
return str(agent_id)
return "default"
# ---- wrap_model_call:Hindsight 动态召回注入 -------------------------
def _build_recall_message(self, request: Any) -> HumanMessage | None:
"""按本次模型请求构建一条 Hindsight 召回消息;无召回内容时返回 None。
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
背景信息而非用户的新输入。
"""
config = self._memory_config or get_memory_config()
if not config.enabled:
return None
user_id = get_effective_user_id()
effective_config = get_effective_memory_config(user_id)
if not effective_config.enabled or not effective_config.injection.enabled:
return None
# 只有 Hindsight 提供动态召回;builtin 走系统提示词。
if effective_config.provider != "hindsight" or not effective_config.injection.include_hindsight:
return None
messages = getattr(request, "messages", None) or []
query = _latest_human_text(messages)
if not query:
return None
runtime = getattr(request, "runtime", None)
thread_id = self._resolve_thread_id(runtime)
try:
manager = build_memory_manager(
user_id=user_id,
agent_id=self._resolve_agent_id(runtime),
thread_id=thread_id,
config=effective_config,
)
recall = manager.prefetch_all(query, thread_id=thread_id or "")
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
logger.debug("Hindsight 召回注入失败 (非致命):%s", exc)
return None
if not recall or not recall.strip():
return None
tag = effective_config.injection.context_tag
fenced = (
f"<{tag}>\n"
"[系统提示:以下是召回的记忆上下文,不是用户的新输入,作为背景信息参考。]\n\n"
f"{recall}\n"
f"</{tag}>"
)
return HumanMessage(content=fenced)
@override
def wrap_model_call(self, request: Any, handler: Any) -> Any:
"""模型调用前临时注入 Hindsight 召回 (同步路径)。"""
recall_msg = self._build_recall_message(request)
if recall_msg is not None:
request = request.override(messages=[*request.messages, recall_msg])
return handler(request)
@override
async def awrap_model_call(self, request: Any, handler: Any) -> Any:
"""模型调用前临时注入 Hindsight 召回 (异步路径)。
召回过程是同步阻塞 (含 Hindsight HTTP),放到线程池里跑以免阻塞事件循环。
"""
recall_msg = await asyncio.to_thread(self._build_recall_message, request)
if recall_msg is not None:
request = request.override(messages=[*request.messages, recall_msg])
return await handler(request)
# ---- after_agent:Hindsight 自动 retain -------------------------------
@override
def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
"""agent 完成后把本轮对话写入 Hindsight 工作 bank。
Returns:
None (本中间件不改状态)。
"""
config = self._memory_config or get_memory_config()
if not config.enabled:
return None
user_id = get_effective_user_id()
effective_config = get_effective_memory_config(user_id)
if effective_config.provider != "hindsight" or not effective_config.hindsight.auto_retain:
return None
messages = state.get("messages", [])
if not messages:
return None
filtered_messages = filter_messages_for_memory(messages)
user_text, assistant_text = _last_exchange(filtered_messages)
if not user_text or not assistant_text:
return None
# 轮次序号按对话历史算出 —— provider 每轮被重建、无法自行计数,
# retain_every_n_turns 的节流必须靠这个外部传入的值。
turn_index = count_user_turns(filtered_messages)
thread_id = self._resolve_thread_id(runtime)
try:
manager = build_memory_manager(
user_id=user_id,
agent_id=self._resolve_agent_id(runtime),
thread_id=thread_id,
config=effective_config,
)
manager.sync_all(
user_text,
assistant_text,
thread_id=thread_id or "",
turn_index=turn_index,
)
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
logger.warning("Hindsight retain 失败:%s", exc)
return None
@@ -131,6 +131,14 @@ def _build_runtime_middlewares(
middlewares.append(SandboxAuditMiddleware())
middlewares.append(ToolErrorHandlingMiddleware())
# Tool / skill call metrics — appended last so it sits *deepest* in the
# wrap_tool_call chain: its handler is the raw tool, so it observes the
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
# an error ToolMessage) and the raw return value.
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
middlewares.append(ToolMetricsMiddleware())
return middlewares
@@ -0,0 +1,319 @@
"""Middleware that records per-tool-call metrics for the admin audit page.
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
- Successes record the wall-clock duration.
- Failures record a coarse ``error_category`` plus a one-line message.
Two failure shapes are captured:
1. **Raised exceptions** — the tool blew up. We record, then re-raise so the
outer :class:`ToolErrorHandlingMiddleware` still converts it into an error
``ToolMessage`` for the model.
2. **Self-handled errors** — search/retrieval tools (Serper, Tavily, Jina, …)
catch their own HTTP 429/403/timeout and *return* an error payload instead
of raising. Those never reach an ``except`` block, so we also inspect the
returned ``ToolMessage``: an ``status == "error"`` message, or content that
looks like an error string, is bucketed too.
This middleware is appended **after** ``ToolErrorHandlingMiddleware`` so it
sits deepest in the ``wrap_tool_call`` chain — its handler is the raw tool, so
it observes the unconverted exception and the raw return value.
Writes go through a best-effort store: a DB outage logs a warning and the
agent runtime keeps running.
"""
from __future__ import annotations
import asyncio
import logging
import re
import time
import uuid
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain_core.messages import ToolMessage
from langgraph.errors import GraphBubbleUp
from langgraph.prebuilt.tool_node import ToolCallRequest
from langgraph.types import Command
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
def _now() -> datetime:
return datetime.now(UTC)
def _trim(text: str, limit: int = 500) -> str:
text = (text or "").strip()
return text if len(text) <= limit else text[: limit - 3] + "..."
def _classify_error(text: str) -> str:
"""Bucket an error string into a coarse, admin-friendly category."""
t = (text or "").lower()
if any(k in t for k in ("429", "rate limit", "ratelimit", "too many request", "quota")):
return "rate_limited"
if any(k in t for k in ("403", "forbidden", "ip ban", "ip block", "blocked", "access denied", "banned")):
return "ip_blocked"
if any(k in t for k in ("401", "unauthorized", "api key", "apikey", "invalid key", "authentication", "credential")):
return "auth_error"
if any(k in t for k in ("timeout", "timed out", "deadline")):
return "timeout"
if any(k in t for k in ("connecterror", "connection error", "connection refused", "network", "dns", "unreachable", "ssl", "proxy")):
return "network_error"
if any(k in t for k in ("404", "not found")):
return "not_found"
if any(k in t for k in ("no results", "no result", "0 results", "empty result")):
return "empty_result"
return "tool_error"
def _content_text(content: Any) -> str:
"""Flatten a tool result's content (string or block list) into text."""
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
return " ".join(parts)
return str(content)
def _looks_like_error(text: str) -> bool:
"""Heuristically detect a self-handled error payload returned as content.
Deliberately conservative — only an explicit ``Error:`` prefix or a JSON
object whose first key is ``error`` counts, so a normal result that merely
mentions the word "error" is not misclassified.
"""
head = (text or "").lstrip()[:64].lower()
return head.startswith("error:") or head.startswith('{"error"') or head.startswith("{'error'")
# Skill tools whose ``name`` argument names one specific Agent Skill directly.
_SKILL_NAME_ARG_TOOLS = {"skill_view", "skill_manage", "skill_archive"}
# Matches a skill directory embedded in any tool-argument path. Skills live
# under ``skills/{public,custom}/{name}/`` — both inside the sandbox
# (``/mnt/skills/...``) and on the host. The lead agent's system prompt tells
# it to load a skill by calling ``read_file`` on that SKILL.md path (it does
# NOT call a dedicated "use skill" tool), so the skill name has to be recovered
# from the path argument of whatever tool touched it (read_file / bash / ls /
# str_replace / write_file ...).
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
def _skill_name_from_call(tool_name: str | None, tool_call: Any) -> str | None:
"""Recover the Agent Skill a tool call targets, or ``None``.
Two detection paths:
1. ``skill_view`` / ``skill_manage`` / ``skill_archive`` take an explicit
``name`` argument.
2. Every other tool references a skill by *path*. The canonical skill
invocation is ``read_file('/mnt/skills/<cat>/<name>/SKILL.md')``; helper
files and ``bash`` script runs hit the same directory. So we scan the
string arguments for the ``skills/{public,custom}/{name}/`` shape.
"""
if not isinstance(tool_call, dict):
return None
args = tool_call.get("args")
if not isinstance(args, dict):
return None
# Case 1 — explicit ``name`` argument on the skill tools.
if tool_name in _SKILL_NAME_ARG_TOOLS:
raw = args.get("name")
if isinstance(raw, str) and raw.strip():
return raw.strip()
# Case 2 — a skill path inside any string argument.
for value in args.values():
if isinstance(value, str):
match = _SKILL_PATH_RE.search(value)
if match:
return match.group(1)
return None
def _evaluate_skill_view(text: str) -> tuple[str, str | None, str | None]:
"""Classify a ``skill_view`` return.
``skill_view`` never raises — on failure it returns a plain string
(``Skill 'x' not found.`` / ``Error reading skill 'x': ...``); on success
it returns the SKILL.md content. So success/failure must be read off the
returned text, not an exception or ``status`` flag.
"""
head = (text or "").lstrip()
low = head[:200].lower()
if head.startswith("Skill '") and "not found" in low:
return "error", "not_found", _trim(text)
if low.startswith("error reading skill"):
return "error", _classify_error(text), _trim(text)
return "success", None, None
def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | None, str | None]:
"""Return ``(status, error_category, error_message)`` for a tool return."""
if isinstance(result, ToolMessage):
text = _content_text(result.content)
if getattr(result, "status", None) == "error":
return "error", _classify_error(text), _trim(text)
# skill_view swallows its own failures into a plain string return.
if tool_name == "skill_view":
return _evaluate_skill_view(text)
if _looks_like_error(text):
return "error", _classify_error(text), _trim(text)
return "success", None, None
# Command (control flow) or any other shape — treat as a successful call.
return "success", None, None
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
"""Wrap each tool call to time it and persist a metric row."""
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._store_override = store
# -- store / context ----------------------------------------------------
def _store(self) -> ToolMetricsStore | None:
if self._store_override is not None:
return self._store_override
try:
sf = get_session_factory()
except Exception:
return None
return make_tool_metrics_store(sf)
@staticmethod
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in keys:
value = context.get(key)
if isinstance(value, str) and value:
return value
config = getattr(runtime, "config", None)
if isinstance(config, dict):
configurable = config.get("configurable") if isinstance(config.get("configurable"), dict) else {}
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
for bucket in (configurable, metadata):
for key in keys:
value = bucket.get(key)
if isinstance(value, str) and value:
return value
return None
def _build_base_record(self, request: ToolCallRequest) -> ToolMetricRecord:
tool_call = getattr(request, "tool_call", None) or {}
tool_name = tool_call.get("name") if isinstance(tool_call, dict) else None
tool_name = str(tool_name) if tool_name else None
return ToolMetricRecord(
id=uuid.uuid4().hex,
created_at=_now(),
user_id=get_effective_user_id(),
thread_id=self._runtime_lookup(request, ("thread_id",)),
run_id=self._runtime_lookup(request, ("run_id", "langgraph_run_id")),
agent_name=self._runtime_lookup(request, ("agent_name", "agent_id")),
skill_name=_skill_name_from_call(tool_name, tool_call),
tool_name=tool_name,
)
# -- finalize -----------------------------------------------------------
def _finalize_result(self, record: ToolMetricRecord, result: Any, started: float) -> None:
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
record.status, record.error_category, record.error_message = _evaluate_result(
result, record.tool_name
)
def _finalize_error(self, record: ToolMetricRecord, exc: BaseException, started: float) -> None:
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
record.status = "error"
name = exc.__class__.__name__
combined = f"{name}: {(str(exc) or name).strip()}"
record.error_category = _classify_error(combined)
record.error_message = _trim(combined)
def _flush(self, record: ToolMetricRecord) -> None:
store = self._store()
if store is None:
return
try:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
# Fire-and-forget — never let metric IO block the tool path.
loop.create_task(store.record(record))
else:
asyncio.run(store.record(record))
except Exception:
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
# -- wrap ---------------------------------------------------------------
@override
def wrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], ToolMessage | Command],
) -> ToolMessage | Command:
record = self._build_base_record(request)
started = time.monotonic()
try:
result = handler(request)
except GraphBubbleUp:
# LangGraph control flow — not a tool failure.
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_result(record, result, started)
self._flush(record)
return result
@override
async def awrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
) -> ToolMessage | Command:
record = self._build_base_record(request)
started = time.monotonic()
try:
result = await handler(request)
except GraphBubbleUp:
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_result(record, result, started)
self._flush(record)
return result
@@ -765,27 +765,22 @@ class DeerFlowClient:
]
}
def get_memory(self) -> dict:
"""Get current memory data.
def get_memory(self, agent: str | None = None) -> dict:
"""读取 Memory V2 本地记忆 (USER.md / MEMORY.md)。
Args:
agent: 可选 agent 名;省略表示默认 agent。
Returns:
Memory data dict (see src/agents/memory/updater.py for structure).
``{user_entries, memory_entries, user_usage, memory_usage}``。
"""
from deerflow.agents.memory.updater import get_memory_data
return get_memory_data(user_id=get_effective_user_id())
def export_memory(self) -> dict:
"""Export current memory data for backup or transfer."""
from deerflow.agents.memory.updater import get_memory_data
return get_memory_data(user_id=get_effective_user_id())
def import_memory(self, memory_data: dict) -> dict:
"""Import and persist full memory data."""
from deerflow.agents.memory.updater import import_memory_data
return import_memory_data(memory_data, user_id=get_effective_user_id())
provider = self._build_memory_provider(agent)
return {
"user_entries": provider.read("user"),
"memory_entries": provider.read("memory"),
"user_usage": provider.usage("user"),
"memory_usage": provider.usage("memory"),
}
def get_model(self, name: str) -> dict | None:
"""Get a specific model's configuration by name.
@@ -949,83 +944,73 @@ class DeerFlowClient:
return get_or_new_skill_storage().install_skill_from_archive(skill_path)
# ------------------------------------------------------------------
# Public API — memory management
# Public API — memory management (Memory V2)
# ------------------------------------------------------------------
def reload_memory(self) -> dict:
"""Reload memory data from file, forcing cache invalidation.
def _build_memory_provider(self, agent: str | None = None):
"""按生效配置构建并初始化一个 V2 builtin provider。"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.config.paths import get_paths
Returns:
The reloaded memory data dict.
"""
from deerflow.agents.memory.updater import reload_memory_data
return reload_memory_data(user_id=get_effective_user_id())
def clear_memory(self) -> dict:
"""Clear all persisted memory data."""
from deerflow.agents.memory.updater import clear_memory_data
return clear_memory_data(user_id=get_effective_user_id())
def create_memory_fact(self, content: str, category: str = "context", confidence: float = 0.5) -> dict:
"""Create a single fact manually."""
from deerflow.agents.memory.updater import create_memory_fact
return create_memory_fact(content=content, category=category, confidence=confidence)
def delete_memory_fact(self, fact_id: str) -> dict:
"""Delete a single fact from memory by fact id."""
from deerflow.agents.memory.updater import delete_memory_fact
return delete_memory_fact(fact_id)
def update_memory_fact(
self,
fact_id: str,
content: str | None = None,
category: str | None = None,
confidence: float | None = None,
) -> dict:
"""Update a single fact manually, preserving omitted fields."""
from deerflow.agents.memory.updater import update_memory_fact
return update_memory_fact(
fact_id=fact_id,
content=content,
category=category,
confidence=confidence,
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
provider = BuiltinFileProvider(
memory_char_limit=config.builtin.memory_char_limit,
user_char_limit=config.builtin.user_char_limit,
deduplicate_on_load=config.builtin.deduplicate_on_load,
security=config.security,
)
provider.initialize(
user_id=user_id,
agent_id=agent or "default",
base_dir=str(get_paths().base_dir),
)
return provider
def add_memory_entry(self, target: str, content: str, agent: str | None = None) -> dict:
"""新增一条记忆条目。``target`` 为 'user'(用户画像) 或 'memory'(工作笔记)。"""
return self._build_memory_provider(agent).add(target, content)
def replace_memory_entry(self, target: str, old_text: str, content: str, agent: str | None = None) -> dict:
"""用 ``old_text`` 短串定位并替换一条记忆条目。"""
return self._build_memory_provider(agent).replace(target, old_text, content)
def remove_memory_entry(self, target: str, old_text: str, agent: str | None = None) -> dict:
"""用 ``old_text`` 短串定位并删除一条记忆条目。"""
return self._build_memory_provider(agent).remove(target, old_text)
def get_memory_config(self) -> dict:
"""Get memory system configuration.
"""读取当前用户的生效记忆配置 (含 per-user 覆盖)。"""
from deerflow.config.memory_config import get_effective_memory_config
return get_effective_memory_config(get_effective_user_id()).model_dump()
def search_memory(self, query: str, mode: str = "recall", agent: str | None = None) -> dict:
"""代理 Hindsight 检索。
Args:
query: 检索查询。
mode: ``recall`` 返回原始片段;``reflect`` 返回 LLM 综合结论。
agent: 可选 agent 名。
Returns:
Memory config dict.
``{available, results, text}``;Hindsight 不可用时 available=False。
"""
from deerflow.config.memory_config import get_memory_config
from deerflow.agents.memory.providers.hindsight import HindsightProvider
from deerflow.config.memory_config import get_effective_memory_config
config = get_memory_config()
return {
"enabled": config.enabled,
"storage_path": config.storage_path,
"debounce_seconds": config.debounce_seconds,
"max_facts": config.max_facts,
"fact_confidence_threshold": config.fact_confidence_threshold,
"injection_enabled": config.injection_enabled,
"max_injection_tokens": config.max_injection_tokens,
}
def get_memory_status(self) -> dict:
"""Get memory status: config + current data.
Returns:
Dict with "config" and "data" keys.
"""
return {
"config": self.get_memory_config(),
"data": self.get_memory(),
}
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
if not config.enabled or config.provider != "hindsight":
return {"available": False, "results": [], "text": ""}
provider = HindsightProvider(config.hindsight)
if not provider.is_available():
return {"available": False, "results": [], "text": ""}
provider.initialize(user_id=user_id, agent_id=agent or "default", thread_id="", base_dir="")
if mode == "reflect":
return {"available": True, "results": [], "text": provider.reflect(query)}
return {"available": True, "results": provider.recall(query), "text": ""}
# ------------------------------------------------------------------
# Public API — file uploads
@@ -43,9 +43,27 @@ logger = logging.getLogger(__name__)
DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest"
DEFAULT_PORT = 8080
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
PACKAGE_MANAGER_ENV_KEYS = (
"PIP_INDEX_URL",
"PIP_EXTRA_INDEX_URL",
"PIP_TRUSTED_HOST",
"PIP_CERT",
"PIP_CLIENT_CERT",
"UV_INDEX_URL",
"UV_EXTRA_INDEX_URL",
"UV_DEFAULT_INDEX",
"UV_INDEX",
"UV_TRUSTED_HOST",
"HTTP_PROXY",
"HTTPS_PROXY",
"NO_PROXY",
"http_proxy",
"https_proxy",
"no_proxy",
)
def _lock_file_exclusive(lock_file) -> None:
@@ -142,9 +160,9 @@ class AioSandboxProvider(SandboxProvider):
Local provider manages container lifecycle directly (start/stop).
"""
provisioner_url = self._config.get("provisioner_url")
if provisioner_url:
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
return RemoteSandboxBackend(provisioner_url=provisioner_url)
if provisioner_url:
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
return RemoteSandboxBackend(provisioner_url=provisioner_url, environment=self._config["environment"])
logger.info("Using local container sandbox backend")
return LocalContainerBackend(
@@ -172,20 +190,24 @@ class AioSandboxProvider(SandboxProvider):
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
"mounts": sandbox_config.mounts or [],
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
"provisioner_url": getattr(sandbox_config, "provisioner_url", None) or "",
}
@staticmethod
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
"""Resolve environment variable references (values starting with $)."""
resolved = {}
for key, value in env_config.items():
if isinstance(value, str) and value.startswith("$"):
env_name = value[1:]
resolved[key] = os.environ.get(env_name, "")
else:
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
"""Resolve environment variable references (values starting with $)."""
resolved = {}
for key in PACKAGE_MANAGER_ENV_KEYS:
value = os.environ.get(key)
if value:
resolved[key] = value
for key, value in env_config.items():
if isinstance(value, str) and value.startswith("$"):
env_name = value[1:]
resolved[key] = os.environ.get(env_name, "")
else:
resolved[key] = str(value)
return resolved
@@ -40,14 +40,17 @@ class RemoteSandboxBackend(SandboxBackend):
provisioner_url: http://provisioner:8002
"""
def __init__(self, provisioner_url: str):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
"""
self._provisioner_url = provisioner_url.rstrip("/")
def __init__(self, provisioner_url: str, environment: dict[str, str] | None = None):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
environment: Environment variables for sandbox pods. The remote
provisioner may ignore this field if it does not support it.
"""
self._provisioner_url = provisioner_url.rstrip("/")
self._environment = environment or {}
@property
def provisioner_url(self) -> str:
@@ -91,10 +94,11 @@ class RemoteSandboxBackend(SandboxBackend):
try:
resp = requests.post(
f"{self._provisioner_url}/api/sandboxes",
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
},
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
"environment": self._environment,
},
timeout=30,
)
resp.raise_for_status()
@@ -11,8 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
No database is initialized.
@@ -21,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
to reference environment variables from .env:
database:
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
"""
from __future__ import annotations
@@ -37,31 +42,40 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres"] = Field(
default="memory",
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
)
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
echo_sql: bool = Field(
default=False,
description="Echo all SQL statements to log (debug only).",
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres only).",
description="Connection pool size for the app ORM engine (postgres/mysql only).",
)
# -- Derived helpers (not user-configured) --
@@ -94,9 +108,16 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
@@ -1,83 +1,315 @@
"""Configuration for memory mechanism."""
from pydantic import BaseModel, Field
class MemoryConfig(BaseModel):
"""Configuration for global memory mechanism."""
enabled: bool = Field(
default=True,
description="Whether to enable memory mechanism",
)
storage_path: str = Field(
default="",
description=(
"Path to store memory data. "
"If empty, defaults to per-user memory at `{base_dir}/users/{user_id}/memory.json`. "
"Absolute paths are used as-is and opt out of per-user isolation "
"(all users share the same file). "
"Relative paths are resolved against `Paths.base_dir` "
"(not the backend working directory). "
"Note: if you previously set this to `.deer-flow/memory.json`, "
"the file will now be resolved as `{base_dir}/.deer-flow/memory.json`; "
"migrate existing data or use an absolute path to preserve the old location."
),
)
storage_class: str = Field(
default="deerflow.agents.memory.storage.FileMemoryStorage",
description="The class path for memory storage provider",
)
debounce_seconds: int = Field(
default=30,
ge=1,
le=300,
description="Seconds to wait before processing queued updates (debounce)",
)
model_name: str | None = Field(
default=None,
description="Model name to use for memory updates (None = use default model)",
)
max_facts: int = Field(
default=100,
ge=10,
le=500,
description="Maximum number of facts to store",
)
fact_confidence_threshold: float = Field(
default=0.7,
ge=0.0,
le=1.0,
description="Minimum confidence threshold for storing facts",
)
injection_enabled: bool = Field(
default=True,
description="Whether to inject memory into system prompt",
)
max_injection_tokens: int = Field(
default=2000,
ge=100,
le=8000,
description="Maximum tokens to use for memory injection",
)
# Global configuration instance
_memory_config: MemoryConfig = MemoryConfig()
def get_memory_config() -> MemoryConfig:
"""Get the current memory configuration."""
return _memory_config
def set_memory_config(config: MemoryConfig) -> None:
"""Set the memory configuration."""
global _memory_config
_memory_config = config
def load_memory_config_from_dict(config_dict: dict) -> None:
"""Load memory configuration from a dictionary."""
global _memory_config
_memory_config = MemoryConfig(**config_dict)
"""Memory V2 - 记忆子系统配置。
配置结构 (详见 docs/MEMORY_V2_DESIGN_ZH.md §9)
==============================================
``MemoryConfig`` 采用嵌套结构::
memory:
enabled: true
provider: hindsight # 外部 provider 选哪个 (builtin 始终启用)
builtin: {...} # 本地 USER.md / MEMORY.md
hindsight: {...} # Hindsight HTTP 客户端
injection: {...} # 系统提示词注入
security: {...} # 内容安全扫描
三层配置覆盖
============
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
4. Pydantic 默认值
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
"""
from __future__ import annotations
import copy
import json
import logging
from typing import Any
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
# ===========================================================================
# 嵌套子配置
# ===========================================================================
class BuiltinMemoryConfig(BaseModel):
"""本地文件 provider (USER.md / MEMORY.md) 配置。"""
enabled: bool = Field(default=True, description="是否启用 builtin 本地记忆 provider")
memory_char_limit: int = Field(
default=2200, ge=200, le=20000, description="MEMORY.md 字符配额"
)
user_char_limit: int = Field(
default=1375, ge=200, le=20000, description="USER.md 字符配额"
)
deduplicate_on_load: bool = Field(
default=True, description="加载时是否对条目去重"
)
class HindsightMemoryConfig(BaseModel):
"""Hindsight 外部 provider 配置。"""
mode: str = Field(
default="local_external",
description="连接模式:cloud / local_embedded / local_external",
)
api_url: str = Field(default="http://localhost:8765", description="Hindsight API 地址")
api_key: str = Field(default="", description="API key,建议用 $HINDSIGHT_API_KEY 环境变量")
bank_id_template: str = Field(
default="deerflow-user-{user_id}",
description="全局 user bank 命名模板,占位符 {user_id}/{agent_id}/{thread_id}",
)
work_bank_id_template: str = Field(
default="deerflow-work-{user_id}-{agent_id}",
description="agent 私有工作 bank 命名模板",
)
memory_mode: str = Field(
default="context",
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
)
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
bank_retain_mission: str = Field(default="", description="引导抽取内容的 mission")
recall_budget: str = Field(default="mid", description="召回深度:low / mid / high")
recall_max_tokens: int = Field(default=4096, ge=256, le=32768, description="召回结果 token 上限")
recall_max_input_chars: int = Field(default=800, ge=64, le=8000, description="召回 query 长度上限")
auto_recall: bool = Field(default=True, description="每轮对话前是否自动召回")
auto_retain: bool = Field(default=True, description="是否自动写入对话")
retain_async: bool = Field(default=True, description="retain 是否在 Hindsight 服务端异步处理")
retain_every_n_turns: int = Field(default=1, ge=1, le=50, description="每 N 轮 retain 一次")
retain_user_prefix: str = Field(default="User", description="自动 retain 时用户发言前缀")
retain_assistant_prefix: str = Field(default="Assistant", description="自动 retain 时助手发言前缀")
class InjectionConfig(BaseModel):
"""系统提示词记忆注入配置。"""
enabled: bool = Field(default=True, description="是否往系统提示词注入记忆")
max_tokens: int = Field(default=2000, ge=100, le=8000, description="注入 token 上限")
include_builtin: bool = Field(default=True, description="是否注入 builtin 本地记忆")
include_hindsight: bool = Field(default=True, description="是否注入 Hindsight 召回结果")
context_tag: str = Field(
default="memory-context", description="包裹外部召回内容的标签名"
)
class SecurityConfig(BaseModel):
"""记忆内容安全配置。"""
scan_content: bool = Field(default=True, description="是否扫描注入/外渗模式")
block_invisible_unicode: bool = Field(default=True, description="是否阻断不可见 unicode")
# ===========================================================================
# 顶层配置
# ===========================================================================
class MemoryConfig(BaseModel):
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
enabled: bool = Field(default=True, description="记忆子系统总开关")
provider: str = Field(
default="builtin",
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
)
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
injection: InjectionConfig = Field(default_factory=InjectionConfig)
security: SecurityConfig = Field(default_factory=SecurityConfig)
# ===========================================================================
# per-user 运行时覆盖
# ===========================================================================
# 用户可在运行时覆盖的字段白名单 (点号路径)。
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
{
"injection.enabled",
"injection.max_tokens",
"injection.include_builtin",
"injection.include_hindsight",
"hindsight.memory_mode",
"hindsight.recall_budget",
"hindsight.bank_mission",
}
)
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
"""
result: dict[str, Any] = {}
def _set_nested(path: str, value: Any) -> None:
if path not in USER_WRITABLE_FIELDS:
logger.warning("忽略非白名单的记忆配置覆盖字段:%s", path)
return
section, field = path.split(".", 1)
result.setdefault(section, {})[field] = value
for key, value in overrides.items():
if isinstance(value, dict):
for sub_key, sub_value in value.items():
_set_nested(f"{key}.{sub_key}", sub_value)
else:
_set_nested(key, value)
return result
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
"""递归合并两个 dict,``override`` 优先。返回新 dict,不修改入参。"""
merged = copy.deepcopy(base)
for key, value in override.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _deep_merge(merged[key], value)
else:
merged[key] = value
return merged
def _load_json_file(path: Any) -> dict[str, Any]:
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
try:
from pathlib import Path
p = Path(path)
if not p.exists():
return {}
data = json.loads(p.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (OSError, json.JSONDecodeError) as exc:
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
return {}
# ===========================================================================
# 全局单例 + 加载入口
# ===========================================================================
_memory_config: MemoryConfig = MemoryConfig()
def get_memory_config() -> MemoryConfig:
"""返回 config.yaml 级别的记忆配置 (不含 per-user 覆盖)。"""
return _memory_config
def set_memory_config(config: MemoryConfig) -> None:
"""设置 config.yaml 级别的记忆配置 (测试与运行时重载用)。"""
global _memory_config
_memory_config = config
def load_memory_config_from_dict(config_dict: dict) -> None:
"""从 dict 加载记忆配置。"""
global _memory_config
_memory_config = MemoryConfig(**config_dict)
def get_effective_memory_config(user_id: str) -> MemoryConfig:
"""按三层覆盖优先级,返回某用户的生效记忆配置。
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
"""
base = _memory_config.model_dump()
try:
from deerflow.config.paths import get_paths
paths = get_paths()
global_overrides = _filter_to_whitelist(
_load_json_file(paths.memory_global_overrides_file)
)
user_overrides = _filter_to_whitelist(
_load_json_file(paths.user_memory_config_file(user_id))
)
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
return _memory_config
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
try:
return MemoryConfig(**merged)
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
return _memory_config
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
current = _load_json_file(path)
filtered = _filter_to_whitelist(patch)
merged = _deep_merge(current, filtered)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
return merged
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
"""撤销某用户的 per-user 覆盖。
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
if field is None:
if path.exists():
path.unlink()
return {}
current = _load_json_file(path)
if "." in field:
section, sub_field = field.split(".", 1)
if isinstance(current.get(section), dict):
current[section].pop(sub_field, None)
if not current[section]:
current.pop(section, None)
else:
current.pop(field, None)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
return current
__all__ = [
"BuiltinMemoryConfig",
"HindsightMemoryConfig",
"InjectionConfig",
"SecurityConfig",
"MemoryConfig",
"USER_WRITABLE_FIELDS",
"get_memory_config",
"set_memory_config",
"load_memory_config_from_dict",
"get_effective_memory_config",
"save_user_memory_override",
"delete_user_memory_override",
]
@@ -155,6 +155,33 @@ class Paths:
"""Per-user per-agent memory: `{base_dir}/users/{user_id}/agents/{id}/memory.json`."""
return self.user_dir(user_id) / "agents" / agent_name.lower() / "memory.json"
# --- Memory V2 布局 (USER.md / MEMORY.md / per-user 配置覆盖) ---
# 详见 docs/MEMORY_V2_DESIGN_ZH.md §5.1。注意与 ``user_md_file`` 区分:
# 后者是 base_dir 下的全局 USER.md (旧布局),这里是 per-user 的 V2 布局。
def user_profile_md_file(self, user_id: str) -> Path:
"""Memory V2 用户画像文件:`{base_dir}/users/{user_id}/USER.md`。
存放 LLM 主动写入的 ``target="user"`` 记忆 (用户身份、偏好,跨 agent 共享)。
"""
return self.user_dir(user_id) / "USER.md"
def user_agent_memory_md_file(self, user_id: str, agent_name: str) -> Path:
"""Memory V2 agent 工作笔记文件:`{base_dir}/users/{user_id}/agents/{id}/MEMORY.md`。
存放 LLM 主动写入的 ``target="memory"`` 记忆 (agent 私有工作笔记)。
"""
return self.user_dir(user_id) / "agents" / agent_name.lower() / "MEMORY.md"
def user_memory_config_file(self, user_id: str) -> Path:
"""Memory V2 per-user 运行时配置覆盖:`{base_dir}/users/{user_id}/memory_config.json`。"""
return self.user_dir(user_id) / "memory_config.json"
@property
def memory_global_overrides_file(self) -> Path:
"""Memory V2 全局配置覆盖 (管理员配):`{base_dir}/memory_user_overrides.json`。"""
return self.base_dir / "memory_user_overrides.json"
def thread_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
"""
Host path for a thread's data.
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class AgentRow(Base):
@@ -16,9 +17,9 @@ class AgentRow(Base):
# Stored as a plain string so the same id can be used as the filesystem
# directory name under {base_dir}/agents/{id}/.
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,11 @@
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
if session_factory is None:
return None
return AIWritingSessionRepository(session_factory)
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
@@ -0,0 +1,30 @@
"""ORM model for AI writing sessions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import PortableLongText
class AIWritingSessionRow(Base):
__tablename__ = "ai_writing_sessions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -0,0 +1,197 @@
"""SQLAlchemy-backed AI writing session storage."""
from __future__ import annotations
import json
from datetime import UTC, datetime
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_UNSET = object()
class AIWritingSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: AIWritingSessionRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
for k in ("completed_interventions", "review_result", "transcript"):
val = d.get(k)
if isinstance(val, str):
try:
d[k] = json.loads(val)
except Exception:
d[k] = None
return d
async def create(
self,
*,
id: str,
title: str,
user_intent: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
now = datetime.now(UTC)
row = AIWritingSessionRow(
id=id,
user_id=resolved_user_id,
title=title,
user_intent=user_intent,
status="in_progress",
draft_title=None,
draft_markdown=None,
completed_interventions=None,
review_result=None,
transcript=None,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
title=_UNSET,
status=_UNSET,
draft_title=_UNSET,
draft_markdown=_UNSET,
completed_interventions=_UNSET,
review_result=_UNSET,
transcript=_UNSET,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
if title is not _UNSET:
row.title = title
if status is not _UNSET:
row.status = status
if draft_title is not _UNSET:
row.draft_title = draft_title
if draft_markdown is not _UNSET:
row.draft_markdown = draft_markdown
if completed_interventions is not _UNSET:
row.completed_interventions = (
json.dumps(completed_interventions, ensure_ascii=False)
if completed_interventions is not None
else None
)
if review_result is not _UNSET:
row.review_result = (
json.dumps(review_result, ensure_ascii=False)
if review_result is not None
else None
)
if transcript is not _UNSET:
row.transcript = (
json.dumps(transcript, ensure_ascii=False)
if transcript is not None
else None
)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
return self._row_to_dict(row)
async def list(
self,
*,
user_id: str | None | _AutoSentinel = AUTO,
limit: int = 50,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
stmt = select(AIWritingSessionRow)
if resolved_user_id is not None:
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return False
if resolved_user_id is not None and row.user_id != resolved_user_id:
return False
await session.delete(row)
await session.commit()
return True
async def list_older_than(
self,
cutoff: datetime,
*,
only_finished: bool = False,
limit: int = 500,
) -> list[str]:
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
Args:
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
only_finished: True 时只清 status in (done/error),False 不看状态
limit: 单批最多返回多少条,用于分批处理避免大事务
"""
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
if only_finished:
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [row[0] for row in result.all()]
async def delete_by_ids(self, session_ids: list[str]) -> int:
"""按 id 列表批量删除。返回实际删除的行数。"""
if not session_ids:
return 0
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
async with self._sf() as session:
result = await session.execute(stmt)
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,11 @@
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.article_types.sql import ArticleTypeRepository
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
if session_factory is None:
return None
return ArticleTypeRepository(session_factory)
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
@@ -0,0 +1,24 @@
"""ORM model for AI writing article types."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class ArticleTypeRow(Base):
__tablename__ = "ai_writing_article_types"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
label: Mapped[str] = mapped_column(String(128), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -0,0 +1,123 @@
"""SQLAlchemy-backed article type storage."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.article_types.model import ArticleTypeRow
_UNSET = object()
class ArticleTypeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: ArticleTypeRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
return d
async def create(
self,
*,
key: str,
label: str,
description: str | None = None,
prompt_hint: str | None = None,
default_word_count: int = 800,
sort_order: int = 0,
) -> dict:
now = datetime.now(UTC)
row = ArticleTypeRow(
id=str(uuid.uuid4()),
key=key,
label=label,
description=description,
prompt_hint=prompt_hint,
default_word_count=default_word_count,
sort_order=sort_order,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
article_type_id: str,
*,
key=_UNSET,
label=_UNSET,
description=_UNSET,
prompt_hint=_UNSET,
default_word_count=_UNSET,
sort_order=_UNSET,
) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
if key is not _UNSET:
row.key = key
if label is not _UNSET:
row.label = label
if description is not _UNSET:
row.description = description
if prompt_hint is not _UNSET:
row.prompt_hint = prompt_hint
if default_word_count is not _UNSET:
row.default_word_count = default_word_count
if sort_order is not _UNSET:
row.sort_order = sort_order
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(self, article_type_id: str) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
return self._row_to_dict(row)
async def get_by_key(self, key: str) -> dict | None:
async with self._sf() as session:
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return None
return self._row_to_dict(row)
async def list(self) -> list[dict]:
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(self, article_type_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_keys(self) -> list[str]:
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return list(result.scalars())
@@ -10,10 +10,13 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import json
import logging
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
import json
import logging
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
def _json_serializer(obj: object) -> str:
@@ -23,11 +26,37 @@ def _json_serializer(obj: object) -> str:
logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
async def _auto_create_postgres_db(url: str) -> None:
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _add_runtime_vendor_path() -> None:
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
The offline Docker image keeps the Python environment inside the image, while
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
unchanged startup script can still import it after container recreation.
"""
try:
from deerflow.config.runtime_paths import runtime_home
vendor_dir = runtime_home() / "python-vendor"
except Exception:
vendor_dir = Path(".deer-flow/python-vendor").resolve()
vendor = str(vendor_dir)
if vendor_dir.exists() and vendor not in sys.path:
sys.path.insert(0, vendor)
def _mysql_driver_name(url: str) -> str:
if url.startswith("mysql+aiomysql://"):
return "aiomysql"
return "asyncmy"
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
The target database name is extracted from *url*. The connection is
@@ -65,8 +94,8 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", or "postgres".
url: SQLAlchemy async URL (for sqlite/postgres).
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
echo: Echo SQL to log.
pool_size: Postgres connection pool size.
sqlite_dir: Directory to create for SQLite (ensured to exist).
@@ -77,11 +106,20 @@ async def init_engine(
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
if find_spec(driver) is None:
raise ImportError(
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if backend == "sqlite":
import os
@@ -112,16 +150,25 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
elif backend == "postgres":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
@@ -162,10 +209,10 @@ async def init_engine_from_config(config) -> None:
await init_engine("memory")
return
await init_engine(
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, Text, UniqueConstraint
from sqlalchemy import String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class FeedbackRow(Base):
@@ -29,4 +30,4 @@ class FeedbackRow(Base):
comment: Mapped[str | None] = mapped_column(Text)
# Optional text feedback from the user
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Float, Integer, String, Text
from sqlalchemy import Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmCallMetricRow(Base):
@@ -18,7 +19,7 @@ class LlmCallMetricRow(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
# sort/filter by recency.
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
@@ -44,12 +44,13 @@ def run_migrations_offline() -> None:
context.run_migrations()
def do_run_migrations(connection):
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True, # Required for SQLite ALTER TABLE support
)
def do_run_migrations(connection):
render_as_batch = connection.dialect.name == "sqlite"
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
)
with context.begin_transaction():
context.run_migrations()
@@ -7,9 +7,11 @@ Create Date: 2026-05-12
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
from deerflow.persistence.types import PortableJSON
revision = "20260512_01"
@@ -23,9 +25,9 @@ def _has_column(table_name: str, column_name: str) -> bool:
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "result_json"):
op.add_column("scheduled_task_runs", sa.Column("result_json", sa.JSON(), nullable=True))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "result_json"):
op.add_column("scheduled_task_runs", sa.Column("result_json", PortableJSON(), nullable=True))
def downgrade() -> None:
@@ -30,7 +30,7 @@ def upgrade() -> None:
op.create_table(
"agents",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("name", sa.String(191), nullable=False),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("published", sa.Boolean(), nullable=False, server_default=sa.false()),
@@ -28,7 +28,7 @@ def upgrade() -> None:
op.create_table(
"tags",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(256), nullable=False),
sa.Column("name", sa.String(159), nullable=False),
sa.Column("tag_type", sa.String(32), nullable=False),
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
@@ -0,0 +1,50 @@
"""Create ai_writing_sessions table.
Revision ID: 20260517_01
Revises: 20260515_01
Create Date: 2026-05-17
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_01"
down_revision = "20260515_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("ai_writing_sessions"):
return
op.create_table(
"ai_writing_sessions",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("title", sa.String(512), nullable=False, server_default=""),
sa.Column("user_intent", sa.String(512), nullable=True),
sa.Column("status", sa.String(32), nullable=False, server_default="in_progress"),
sa.Column("draft_title", sa.String(512), nullable=True),
sa.Column("draft_markdown", sa.Text, nullable=True),
sa.Column("completed_interventions", sa.Text, nullable=True),
sa.Column("review_result", sa.Text, nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_ai_writing_sessions_user_id", "ai_writing_sessions", ["user_id"])
op.create_index("ix_ai_writing_sessions_updated_at", "ai_writing_sessions", ["updated_at"])
def downgrade() -> None:
if _has_table("ai_writing_sessions"):
op.drop_table("ai_writing_sessions")
@@ -0,0 +1,117 @@
"""Create ai_writing_article_types table with default types.
Revision ID: 20260517_02
Revises: 20260517_01
Create Date: 2026-05-17
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_02"
down_revision = "20260517_01"
branch_labels = None
depends_on = None
_DEFAULT_TYPES = [
{
"key": "news",
"label": "新闻资讯",
"description": "时效性强的新闻报道,注重事实准确、简洁客观",
"prompt_hint": "以新闻报道的风格写作,注重5W1H要素,语言简洁客观,突出时效性和事实准确性",
"default_word_count": 600,
"sort_order": 0,
},
{
"key": "analysis",
"label": "分析报道",
"description": "深入分析事件背景与影响,提供专业见解",
"prompt_hint": "以分析报道的风格写作,提供深度背景分析、多角度解读和专业见解,逻辑清晰",
"default_word_count": 1200,
"sort_order": 1,
},
{
"key": "feature",
"label": "深度特稿",
"description": "人物或事件的深度叙事,注重细节与故事性",
"prompt_hint": "以深度特稿的风格写作,注重叙事性和细节描写,人物刻画生动,故事性强,情感真实",
"default_word_count": 2000,
"sort_order": 2,
},
{
"key": "report",
"label": "调研报告",
"description": "数据驱动的专题研究报告,结构严谨",
"prompt_hint": "以调研报告的风格写作,数据详实,结构严谨(摘要/背景/发现/结论),语言专业规范",
"default_word_count": 3000,
"sort_order": 3,
},
]
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("ai_writing_article_types"):
op.create_table(
"ai_writing_article_types",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("key", sa.String(64), nullable=False, unique=True),
sa.Column("label", sa.String(128), nullable=False),
sa.Column("description", sa.Text, nullable=True),
sa.Column("prompt_hint", sa.Text, nullable=True),
sa.Column("default_word_count", sa.Integer, nullable=False, server_default="800"),
sa.Column("sort_order", sa.Integer, nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
# 预置默认文章类型(幂等,跳过已存在的 key)。
# 用 SQLAlchemy Core 构造而非裸 SQL:``key`` 是 MySQL 保留字,裸
# ``SELECT key`` 会触发 1064 语法错误。Core 编译器会按方言自动给保留字
# 加引号(MySQL 反引号 / Postgres 双引号),保持跨后端可移植。
article_types = sa.table(
"ai_writing_article_types",
sa.column("id", sa.String),
sa.column("key", sa.String),
sa.column("label", sa.String),
sa.column("description", sa.Text),
sa.column("prompt_hint", sa.Text),
sa.column("default_word_count", sa.Integer),
sa.column("sort_order", sa.Integer),
sa.column("created_at", sa.DateTime),
sa.column("updated_at", sa.DateTime),
)
conn = op.get_bind()
existing_keys = {row[0] for row in conn.execute(sa.select(article_types.c.key)).fetchall()}
now = datetime.now(UTC)
for t in _DEFAULT_TYPES:
if t["key"] not in existing_keys:
conn.execute(
article_types.insert().values(
id=str(uuid.uuid4()),
key=t["key"],
label=t["label"],
description=t["description"],
prompt_hint=t["prompt_hint"],
default_word_count=t["default_word_count"],
sort_order=t["sort_order"],
created_at=now,
updated_at=now,
)
)
def downgrade() -> None:
if _has_table("ai_writing_article_types"):
op.drop_table("ai_writing_article_types")
@@ -0,0 +1,39 @@
"""Add transcript column to ai_writing_sessions.
存储前端组装的完整对话时间线(progressEvents + completedInterventions),
供历史会话回显出与实时对话一致的富时间线。
Revision ID: 20260517_03
Revises: 20260517_02
Create Date: 2026-05-17
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260517_03"
down_revision = "20260517_02"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if _has_column("ai_writing_sessions", "transcript"):
return
op.add_column("ai_writing_sessions", sa.Column("transcript", sa.Text, nullable=True))
def downgrade() -> None:
if _has_column("ai_writing_sessions", "transcript"):
op.drop_column("ai_writing_sessions", "transcript")
@@ -0,0 +1,42 @@
"""Add detail column to skills table.
Revision ID: 20260518_01
Revises: 20260515_01
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_01"
down_revision = "20260515_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
# ``skills`` may not exist yet on a brand-new DB — create_all handles that
# case with the column already present, so only patch existing tables.
if _has_table("skills") and not _has_column("skills", "detail"):
op.add_column("skills", sa.Column("detail", sa.Text(), nullable=True))
def downgrade() -> None:
if _has_column("skills", "detail"):
op.drop_column("skills", "detail")
@@ -0,0 +1,53 @@
"""Create thread_shares table (per-conversation share links).
Revision ID: 20260518_02
Revises: 20260518_01
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_02"
down_revision = "20260518_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
op.create_table(
"thread_shares",
sa.Column("share_code", sa.String(32), primary_key=True),
sa.Column("owner_user_id", sa.String(64), nullable=False),
sa.Column("thread_id", sa.String(64), nullable=False),
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
elif not _has_column("thread_shares", "thread_id"):
# An earlier all-conversations variant of this table — add the column.
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
if _has_table("thread_shares"):
op.drop_table("thread_shares")
@@ -0,0 +1,57 @@
"""Ensure thread_shares.thread_id exists (per-conversation share links).
Standalone migration so databases that already applied an earlier
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
column) get the column added. Idempotent — a no-op when the column is
already present.
Revision ID: 20260518_03
Revises: 20260518_02
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_03"
down_revision = "20260518_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
# Fresh database — 20260518_02 already created the table with thread_id.
return
if not _has_column("thread_shares", "thread_id"):
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
# Leave the column in place — dropping it would break the share feature.
pass
@@ -0,0 +1,39 @@
"""Add retry_count to scheduled_task_runs.
Tracks how many recovery attempts a run has consumed so the scheduler can
cap retries on stuck/timed-out runs.
Revision ID: 20260518_04
Revises: 20260518_03
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_04"
down_revision = "20260518_03"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "retry_count"):
op.add_column(
"scheduled_task_runs",
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
if _has_column("scheduled_task_runs", "retry_count"):
op.drop_column("scheduled_task_runs", "retry_count")
@@ -0,0 +1,44 @@
"""Add mode to thread_shares.
Distinguishes "import" shares (recipient copies the conversation into their
account) from "view" shares (a public, read-only static page).
Revision ID: 20260518_05
Revises: 20260518_04
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_05"
down_revision = "20260518_04"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return True # table missing entirely — skip (created with the column)
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("thread_shares", "mode"):
op.add_column(
"thread_shares",
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if "thread_shares" in inspector.get_table_names() and any(
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
):
op.drop_column("thread_shares", "mode")
@@ -0,0 +1,28 @@
"""Merge the ai-writing and thread-shares migration heads.
Two independent migration branches diverged from ``20260515_01``: the AI
writing feature (``20260517_01..03``) and the thread-shares / scheduled-task
work (``20260518_01..05``). Merging the two branches into one repo left Alembic
with multiple heads, so ``upgrade head`` failed. This is a no-op merge revision
that joins both lineages — it changes no schema.
Revision ID: 20260518_06
Revises: 20260517_03, 20260518_05
Create Date: 2026-05-18
"""
from __future__ import annotations
revision = "20260518_06"
down_revision = ("20260517_03", "20260518_05")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,58 @@
"""Create tool_call_metrics table.
Revision ID: 20260518_07
Revises: 20260518_06
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_07"
down_revision = "20260518_06"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("tool_call_metrics"):
return
op.create_table(
"tool_call_metrics",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("thread_id", sa.String(64), nullable=True),
sa.Column("run_id", sa.String(64), nullable=True),
sa.Column("agent_name", sa.String(128), nullable=True),
sa.Column("tool_name", sa.String(128), nullable=True),
sa.Column("skill_name", sa.String(128), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
sa.Column("error_category", sa.String(32), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_tool_call_metrics_created_at", "tool_call_metrics", ["created_at"])
op.create_index("ix_tool_call_metrics_user_id", "tool_call_metrics", ["user_id"])
op.create_index("ix_tool_call_metrics_thread_id", "tool_call_metrics", ["thread_id"])
op.create_index("ix_tool_call_metrics_run_id", "tool_call_metrics", ["run_id"])
op.create_index("ix_tool_call_metrics_agent_name", "tool_call_metrics", ["agent_name"])
op.create_index("ix_tool_call_metrics_tool_name", "tool_call_metrics", ["tool_name"])
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
op.create_index("ix_tool_call_metrics_status", "tool_call_metrics", ["status"])
op.create_index("ix_tool_call_metrics_error_category", "tool_call_metrics", ["error_category"])
def downgrade() -> None:
if _has_table("tool_call_metrics"):
op.drop_table("tool_call_metrics")
@@ -0,0 +1,62 @@
"""Add skill_name column to tool_call_metrics.
``tool_call_metrics`` is created by ``create_all`` at startup. On deployments
where the table was materialised *before* the ``skill_name`` column existed,
neither ``create_all`` (it never ALTERs) nor revision ``20260518_07`` (it skips
when the table already exists) adds the column — leaving MySQL/Postgres with a
schema that the ORM model expects a column it does not have. This revision
patches the existing table.
Revision ID: 20260519_01
Revises: 20260518_07
Create Date: 2026-05-19
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260519_01"
down_revision = "20260518_07"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
# Brand-new DBs: the table is created with the column already present
# (by create_all / revision 20260518_07) — nothing to patch.
if not _has_table("tool_call_metrics"):
return
if not _has_column("tool_call_metrics", "skill_name"):
op.add_column("tool_call_metrics", sa.Column("skill_name", sa.String(128), nullable=True))
if not _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
def downgrade() -> None:
if _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.drop_index("ix_tool_call_metrics_skill_name", table_name="tool_call_metrics")
if _has_column("tool_call_metrics", "skill_name"):
op.drop_column("tool_call_metrics", "skill_name")
@@ -0,0 +1,64 @@
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
The ``transcript`` column stores the full AI-writing conversation timeline as a
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
a transcript with many progress events overflows it and the save endpoint
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
Revision ID: 20260519_02
Revises: 20260519_01
Create Date: 2026-05-19
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260519_02"
down_revision = "20260519_01"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return False
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
bind = op.get_bind()
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=LONGTEXT(),
existing_nullable=True,
)
def downgrade() -> None:
bind = op.get_bind()
if bind.dialect.name != "mysql":
return
if not _has_column("ai_writing_sessions", "transcript"):
return
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
# is a developer-only path.
import sqlalchemy as sa
op.alter_column(
"ai_writing_sessions",
"transcript",
type_=sa.Text(),
existing_nullable=True,
)
@@ -32,6 +32,8 @@ from deerflow.persistence.scheduled_tasks.model import (
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
__all__ = [
@@ -51,5 +53,7 @@ __all__ = [
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UserRow",
]
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
from sqlalchemy import Index, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class RunEventRow(Base):
@@ -24,9 +25,9 @@ class RunEventRow(Base):
category: Mapped[str] = mapped_column(String(16), nullable=False)
# "message" | "trace" | "lifecycle"
content: Mapped[str] = mapped_column(Text, default="")
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
event_metadata: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
seq: Mapped[int] = mapped_column(nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
__table_args__ = (
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
@@ -5,10 +5,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import JSON, Boolean, DateTime, String, Text
from sqlalchemy import Boolean, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class NotificationRow(Base):
@@ -19,6 +20,6 @@ class NotificationRow(Base):
type: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
payload: Mapped[dict[str, Any]] = mapped_column(PortableJSON(), nullable=False, default=dict)
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True)
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class RecommendedQuestionRow(Base):
@@ -17,5 +18,5 @@ class RecommendedQuestionRow(Base):
title: Mapped[str] = mapped_column(String(256), nullable=False)
content: Mapped[str] = mapped_column(Text, nullable=False)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, Index, String, Text
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class RunRow(Base):
@@ -22,8 +23,8 @@ class RunRow(Base):
model_name: Mapped[str | None] = mapped_column(String(128))
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
kwargs_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
error: Mapped[str | None] = mapped_column(Text)
# Convenience fields (for listing pages without querying RunEventStore)
@@ -43,7 +44,7 @@ class RunRow(Base):
# Follow-up association
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
raise NotImplementedError
# --- Admin (cross-user) ---
@abstractmethod
async def list_all_tasks(self) -> list[dict[str, Any]]:
"""Return every task across all users. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def delete_task_any(self, task_id: str) -> bool:
"""Delete a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs for a task without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs still in a non-terminal state whose ``started_at`` is older
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
or were orphaned by a service restart."""
raise NotImplementedError
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@@ -56,6 +85,11 @@ class ScheduledTaskStore(ABC):
async def delete_run(self, run_id: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def delete_run_any(self, run_id: str) -> bool:
"""Delete a run by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
raise NotImplementedError
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
due.sort(key=lambda task: task["next_run_at"])
return due[:limit]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
return [dict(task) for task in self._tasks.values()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
if task is None:
return None
task.update(data)
return dict(task)
async def delete_task_any(self, task_id: str) -> bool:
return self._tasks.pop(task_id, None) is not None
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
return runs[:limit]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
stuck = [
dict(run)
for run in self._runs.values()
if run.get("status") in ("running", "pending")
and run.get("started_at") is not None
and run["started_at"] <= cutoff
]
stuck.sort(key=lambda run: run["started_at"])
return stuck[:limit]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
key = (data["task_id"], data["scheduled_for"].isoformat())
if key in self._runs_by_key:
@@ -90,11 +121,21 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
run = self._runs.get(run_id)
if run is None or run["user_id"] != user_id:
return False
self._runs.pop(run_id, None)
self._drop_run(run)
return True
async def delete_run_any(self, run_id: str) -> bool:
run = self._runs.get(run_id)
if run is None:
return False
self._drop_run(run)
return True
def _drop_run(self, run: dict[str, Any]) -> None:
self._runs.pop(run["id"], None)
scheduled_for = run.get("scheduled_for")
if hasattr(scheduled_for, "isoformat"):
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class SchedulerThreadRow(Base):
@@ -15,8 +16,8 @@ class SchedulerThreadRow(Base):
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRow(Base):
@@ -29,20 +30,20 @@ class ScheduledTaskRow(Base):
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
name: Mapped[str] = mapped_column(String(256))
name: Mapped[str] = mapped_column(String(127))
prompt: Mapped[str] = mapped_column(Text)
schedule_text: Mapped[str] = mapped_column(String(256))
cron_expr: Mapped[str] = mapped_column(String(128))
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), index=True)
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
last_status: Mapped[str | None] = mapped_column(String(32))
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
execution_context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
published: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskSubscriptionRow(Base):
@@ -53,8 +54,8 @@ class ScheduledTaskSubscriptionRow(Base):
task_id: Mapped[str] = mapped_column(String(64), index=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
notify_element: Mapped[bool] = mapped_column(Boolean, default=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRunRow(Base):
@@ -66,21 +67,25 @@ class ScheduledTaskRunRow(Base):
user_id: Mapped[str] = mapped_column(String(64), index=True)
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), index=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
status: Mapped[str] = mapped_column(String(32), default="pending")
error: Mapped[str | None] = mapped_column(Text)
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
result_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
# Number of recovery attempts made after the first try. A run is retried
# when it overruns the per-attempt timeout or is found orphaned in the
# "running" state (e.g. after a service restart). See ScheduledTaskService.
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskDeliveryProfileRow(Base):
__tablename__ = "scheduled_task_delivery_profiles"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
element_user_id: Mapped[str | None] = mapped_column(String(191), index=True)
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
)
return [_to_dict(row) for row in result.scalars()]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
return [_to_dict(row) for row in result.scalars()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return None
for key, value in data.items():
attr = "execution_context_json" if key == "execution_context" else key
if hasattr(row, attr):
setattr(row, attr, value)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def delete_task_any(self, task_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.task_id == task_id)
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(
ScheduledTaskRunRow.status.in_(("running", "pending")),
ScheduledTaskRunRow.started_at.is_not(None),
ScheduledTaskRunRow.started_at <= cutoff,
)
.order_by(ScheduledTaskRunRow.started_at.asc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
row = ScheduledTaskRunRow(
id=data["id"],
@@ -180,6 +234,15 @@ class ScheduledTaskRepository(ScheduledTaskStore):
await session.commit()
return True
async def delete_run_any(self, run_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRunRow, run_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
@@ -232,7 +295,14 @@ class ScheduledTaskRepository(ScheduledTaskStore):
result = await session.execute(
select(ScheduledTaskRow)
.where(ScheduledTaskRow.published.is_(True))
.order_by(ScheduledTaskRow.published_at.desc().nullslast(), ScheduledTaskRow.created_at.desc())
# ``NULLS LAST`` is unsupported by MySQL — sort by an
# ``IS NULL`` flag instead (False<True ⇒ non-null rows first),
# which is portable across mysql/sqlite/postgres.
.order_by(
ScheduledTaskRow.published_at.is_(None),
ScheduledTaskRow.published_at.desc(),
ScheduledTaskRow.created_at.desc(),
)
.offset(max(0, offset))
.limit(max(1, limit))
)
@@ -69,6 +69,7 @@ class MemorySkillStore(SkillStore):
"name": name,
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", False)),
"detail": data.get("detail"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -83,6 +84,7 @@ class MemorySkillStore(SkillStore):
"name": name,
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", True)),
"detail": data.get("detail"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -98,6 +100,8 @@ class MemorySkillStore(SkillStore):
return None
if "published" in data:
record["published"] = bool(data["published"])
if "detail" in data:
record["detail"] = data["detail"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -109,6 +113,8 @@ class MemorySkillStore(SkillStore):
record["published"] = bool(data["published"])
if "owner_user_id" in data:
record["owner_user_id"] = data["owner_user_id"]
if "detail" in data:
record["detail"] = data["detail"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -10,7 +10,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String
from sqlalchemy import Boolean, DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,5 +24,10 @@ class SkillRow(Base):
name: Mapped[str] = mapped_column(String(64), primary_key=True)
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# Human-authored long-form detail/introduction, editable by the skill owner
# or an admin. Distinct from the skill ``description`` parsed from SKILL.md;
# kept in the DB so admins can annotate built-in skills without rewriting
# the shared on-disk SKILL.md file.
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -63,6 +63,7 @@ class SkillRepository(SkillStore):
name=data["name"],
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", False)),
detail=data.get("detail"),
created_at=now,
updated_at=now,
)
@@ -81,6 +82,7 @@ class SkillRepository(SkillStore):
name=data["name"],
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", True)),
detail=data.get("detail"),
created_at=now,
updated_at=now,
)
@@ -101,6 +103,8 @@ class SkillRepository(SkillStore):
return None
if "published" in data:
row.published = bool(data["published"])
if "detail" in data:
row.detail = data["detail"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -115,6 +119,8 @@ class SkillRepository(SkillStore):
row.published = bool(data["published"])
if "owner_user_id" in data:
row.owner_user_id = data["owner_user_id"]
if "detail" in data:
row.detail = data["detail"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -14,10 +14,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, String, UniqueConstraint
from sqlalchemy import String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
# Allowed tag/target types. Kept here so persistence + API share one source.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
@@ -28,12 +29,12 @@ class TagRow(Base):
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), nullable=False)
name: Mapped[str] = mapped_column(String(159), nullable=False)
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class TagAssignmentRow(Base):
@@ -44,4 +45,4 @@ class TagAssignmentRow(Base):
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -4,10 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, String
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class ThreadMetaRow(Base):
@@ -18,6 +19,6 @@ class ThreadMetaRow(Base):
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
display_name: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(20), default="idle")
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,19 @@
"""Persistence layer for conversation-history share links."""
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
__all__ = [
"ThreadShareStore",
"MemoryThreadShareStore",
"ThreadShareRepository",
"generate_share_code",
"make_thread_share_store",
]
def make_thread_share_store(session_factory):
if session_factory is None:
return MemoryThreadShareStore()
return ThreadShareRepository(session_factory)
@@ -0,0 +1,50 @@
"""Abstract persistence API for per-conversation share links."""
from __future__ import annotations
import secrets
from abc import ABC, abstractmethod
from typing import Any
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
def generate_share_code(length: int = 8) -> str:
"""Return a random, unguessable, human-readable share code."""
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
class ThreadShareStore(ABC):
@abstractmethod
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
"""Create a new share link for one conversation and return its record.
``mode`` is ``"import"`` (recipient copies the conversation) or
``"view"`` (a public read-only static page).
"""
raise NotImplementedError
@abstractmethod
async def get(self, share_code: str) -> dict[str, Any] | None:
"""Look up a share by code regardless of owner (None if missing)."""
raise NotImplementedError
@abstractmethod
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
Used so re-sharing the same conversation in the same mode reuses its
existing code instead of minting a new one each time.
"""
raise NotImplementedError
@abstractmethod
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
"""List all share links created by ``owner_user_id`` (newest first)."""
raise NotImplementedError
@abstractmethod
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
"""Revoke a share link. Returns False if missing or not owned."""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
class MemoryThreadShareStore(ThreadShareStore):
def __init__(self) -> None:
self._shares: dict[str, dict[str, Any]] = {}
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
code = generate_share_code()
while code in self._shares:
code = generate_share_code()
record = {
"share_code": code,
"owner_user_id": owner_user_id,
"thread_id": thread_id,
"revoked": False,
"mode": mode,
"created_at": datetime.now(UTC).isoformat(),
}
self._shares[code] = record
return copy.deepcopy(record)
async def get(self, share_code: str) -> dict[str, Any] | None:
record = self._shares.get(share_code)
return copy.deepcopy(record) if record is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
matches = [
r
for r in self._shares.values()
if r["owner_user_id"] == owner_user_id
and r["thread_id"] == thread_id
and r.get("mode", "import") == mode
and not r["revoked"]
]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
rows.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in rows]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
record = self._shares.get(share_code)
if record is None or record["owner_user_id"] != owner_user_id:
return False
record["revoked"] = True
return True
@@ -0,0 +1,37 @@
"""ORM model for conversation share links.
A share link is a short, revocable code created by a user for *one* of their
conversations. Redeeming it copies that single conversation into the redeemer's
account. The table only stores the link itself — the conversation is copied
lazily at import time.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ThreadShareRow(Base):
__tablename__ = "thread_shares"
# Short, URL-safe, unguessable code embedded in the share link.
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# The user who created the share.
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# The single conversation (thread) this link shares.
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Soft-disable: a revoked link can no longer be previewed or imported.
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Share mode:
# "import" — recipient (a logged-in user) copies the conversation into
# their own account and can keep chatting in the copy.
# "view" — a public, read-only static page anyone can open without
# logging in; viewers cannot send new messages.
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,80 @@
"""SQLAlchemy-backed per-conversation share-link repository."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.model import ThreadShareRow
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
data = row.to_dict()
created = data.get("created_at")
if isinstance(created, datetime):
data["created_at"] = created.isoformat()
return data
class ThreadShareRepository(ThreadShareStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
# Retry on the (astronomically unlikely) code collision.
async with self._sf() as session:
for _ in range(8):
code = generate_share_code()
if await session.get(ThreadShareRow, code) is not None:
continue
row = ThreadShareRow(
share_code=code,
owner_user_id=owner_user_id,
thread_id=thread_id,
revoked=False,
mode=mode,
)
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
raise RuntimeError("Failed to allocate a unique share code")
async def get(self, share_code: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
return _row_to_dict(row) if row is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
stmt = (
select(ThreadShareRow)
.where(
ThreadShareRow.owner_user_id == owner_user_id,
ThreadShareRow.thread_id == thread_id,
ThreadShareRow.mode == mode,
ThreadShareRow.revoked.is_(False),
)
.order_by(ThreadShareRow.created_at.desc())
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalars().first()
return _row_to_dict(row) if row is not None else None
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
if row is None or row.owner_user_id != owner_user_id:
return False
row.revoked = True
await session.commit()
return True
@@ -0,0 +1,35 @@
"""Persistence layer for per-tool-call (skill-call) metrics.
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
writes one row recording duration and outcome. Failures are bucketed into a
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
network_error / not_found / empty_result / tool_error) so administrators can
spot retrieval outages, blocked IPs, and quota problems at a glance.
**Skill-call merging**: a skill is often touched by several tool calls inside
one run (read ``SKILL.md``, run a script, retry after a failure). Rather than
one row per call, ``SqlToolMetricsStore.record`` upserts a single row per
``(run_id, skill_name)``: any success in the run wins (a skill that failed then
finally succeeded is recorded as one success), and a skill that only ever
failed yields one ``error`` row whose ``error_message`` accumulates every
attempt's message. Non-skill tool calls keep one row each.
"""
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
__all__ = [
"ToolCallMetricRow",
"ToolMetricRecord",
"ToolMetricsStore",
"SqlToolMetricsStore",
"make_tool_metrics_store",
]
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlToolMetricsStore(session_factory)
@@ -0,0 +1,84 @@
"""Abstract repository contract for tool / skill call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class ToolMetricRecord:
"""In-memory shape of a tool-call metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
tool_name: str | None = None
# The specific Agent Skill this call targets, when the tool is a skill tool
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
# call's ``name`` argument. Null for every non-skill tool call. This is
# what lets the admin see "skill X was invoked, and succeeded/failed".
skill_name: str | None = None
duration_ms: int = 0
status: str = "success"
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
# network_error / not_found / empty_result / tool_error. Null on success.
error_category: str | None = None
error_message: str | None = None
@dataclass
class ToolMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
tool_name: str | None = None
# When True, keep only rows that target a specific Agent Skill
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
skill_only: bool = False
# Filter to one specific skill by name.
skill_name: str | None = None
status: str | None = None
error_category: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class ToolMetricsStore(ABC):
"""Persistence contract for the tool-metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: ToolMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: ToolMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: ToolMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@abstractmethod
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
Only rows with a non-null ``skill_name`` are counted, so this answers
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""ORM model for per-tool-call (skill-call) metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ToolCallMetricRow(Base):
"""One row per completed (or failed) tool / skill invocation.
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
failures — search tools that swallow an HTTP 429/403 and return an error
payload instead of raising — so an admin can see "retrieval failed" /
"IP blocked" events that never surfaced as exceptions.
"""
__tablename__ = "tool_call_metrics"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
# The specific Agent Skill targeted by a skill tool call (from the call's
# ``name`` argument); null for non-skill tool calls.
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
# "success" or "error".
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
# Coarse failure bucket; null on success.
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"tool_name": self.tool_name,
"skill_name": self.skill_name,
"duration_ms": self.duration_ms,
"status": self.status,
"error_category": self.error_category,
"error_message": self.error_message,
}
@@ -0,0 +1,247 @@
"""SQLAlchemy-backed tool metrics repository."""
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
logger = logging.getLogger(__name__)
# Separator joining every failed attempt's message inside one merged skill row.
_ERROR_SEP = "\n--- \n"
# Hard cap on the accumulated ``error_message`` of a merged row.
_MERGED_ERROR_LIMIT = 2000
# A single skill invocation often spans several tool calls (read SKILL.md, run
# a script, ...), and the model may retry a skill within the same run. Those
# fire-and-forget writes can interleave on the event loop, so the read → merge →
# write of a skill row is guarded by a lock. ``asyncio.Lock`` binds to the loop
# it is used on; the lock is therefore (re)created whenever the running loop
# changes (the Gateway keeps one persistent loop, while the ``asyncio.run``
# fallback path spins up a throwaway loop per call — already serialized).
_merge_lock: asyncio.Lock | None = None
_merge_lock_loop: object = None
def _skill_merge_lock() -> asyncio.Lock:
global _merge_lock, _merge_lock_loop
loop = asyncio.get_running_loop()
if _merge_lock is None or _merge_lock_loop is not loop:
_merge_lock = asyncio.Lock()
_merge_lock_loop = loop
return _merge_lock
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
def _merge_error_messages(existing: str | None, new: str | None) -> str | None:
"""Append a failed attempt's message to the accumulated error text.
All failures are kept ("都统计下来"), separated and capped at
:data:`_MERGED_ERROR_LIMIT` characters.
"""
new = (new or "").strip()
if not new:
return existing
existing = (existing or "").strip()
if not existing:
return new
if new in existing:
# Identical retry failure — no point storing the same line twice.
return existing
combined = existing + _ERROR_SEP + new
if len(combined) > _MERGED_ERROR_LIMIT:
combined = combined[: _MERGED_ERROR_LIMIT - 3] + "..."
return combined
def _merge_skill_row(existing: ToolCallMetricRow, metric: ToolMetricRecord) -> None:
"""Fold another same-run, same-skill call into an existing metric row.
Outcome rules:
- any success in the run -> the row becomes a single ``success`` and earlier
failures are dropped ("先报错、最后成功 -> 记一次成功");
- all calls failed -> a single ``error`` row whose ``error_message``
accumulates every attempt's message ("一直报错 -> 记一次,报错信息都统计下来").
"""
existing.duration_ms = (existing.duration_ms or 0) + (metric.duration_ms or 0)
if metric.status == "success":
if existing.status != "success":
existing.status = "success"
existing.error_category = None
existing.error_message = None
return
# metric.status == "error"
if existing.status == "success":
# The skill already succeeded once this run — keep that outcome.
return
existing.status = "error"
if metric.error_category:
existing.error_category = metric.error_category
existing.error_message = _merge_error_messages(existing.error_message, metric.error_message)
def _row_from_metric(metric: ToolMetricRecord) -> ToolCallMetricRow:
return ToolCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
tool_name=metric.tool_name,
skill_name=metric.skill_name,
duration_ms=metric.duration_ms,
status=metric.status,
error_category=metric.error_category,
error_message=metric.error_message,
)
class SqlToolMetricsStore(ToolMetricsStore):
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: ToolMetricRecord) -> None:
# Skill calls are merged per run: the model may touch one skill across
# several tool calls / retries, and the admin wants a single row per
# (run, skill) instead of one per call. Non-skill tool calls, and skill
# calls with no run_id, fall back to a plain insert.
if metric.skill_name and metric.run_id:
await self._record_skill_merged(metric)
return
try:
async with self._sf() as session:
session.add(_row_from_metric(metric))
await session.commit()
except Exception:
# Metrics are best-effort; never surface a write failure to the runtime.
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
async def _record_skill_merged(self, metric: ToolMetricRecord) -> None:
"""Upsert a per-(run, skill) row, folding repeated calls into one."""
try:
async with _skill_merge_lock():
async with self._sf() as session:
existing = await session.scalar(
select(ToolCallMetricRow)
.where(
ToolCallMetricRow.run_id == metric.run_id,
ToolCallMetricRow.skill_name == metric.skill_name,
)
.order_by(ToolCallMetricRow.created_at)
.limit(1)
)
if existing is None:
session.add(_row_from_metric(metric))
else:
_merge_skill_row(existing, metric)
await session.commit()
except Exception:
logger.warning(
"Failed to merge tool_call_metrics for skill %s (run %s)",
metric.skill_name,
metric.run_id,
exc_info=True,
)
def _apply_filters(self, stmt, query: ToolMetricsQuery):
if query.user_id:
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
if query.tool_name:
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
if query.skill_only:
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
if query.skill_name:
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
if query.status:
stmt = stmt.where(ToolCallMetricRow.status == query.status)
if query.error_category:
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
if query.since is not None:
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
stmt = select(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [row.to_dict() for row in result.scalars()]
async def count(self, query: ToolMetricsQuery) -> int:
stmt = select(func.count()).select_from(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
page_size = 500
offset = 0
while True:
page = await self.list(
ToolMetricsQuery(
user_id=query.user_id,
tool_name=query.tool_name,
skill_only=query.skill_only,
skill_name=query.skill_name,
status=query.status,
error_category=query.error_category,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
# Count success/error per skill. Force the skill-only filter so rows
# without a skill_name (plain tool calls) never leak into the result.
scoped = ToolMetricsQuery(
user_id=query.user_id,
skill_only=True,
skill_name=query.skill_name,
since=query.since,
until=query.until,
)
stmt = select(
ToolCallMetricRow.skill_name,
ToolCallMetricRow.status,
func.count().label("n"),
)
stmt = self._apply_filters(stmt, scoped).group_by(
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
)
agg: dict[str, dict[str, int]] = {}
async with self._sf() as session:
for skill_name, status, n in (await session.execute(stmt)).all():
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
bucket["total"] += n
bucket["success" if status == "success" else "error"] += n
return [
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]
@@ -0,0 +1,105 @@
"""Portable SQLAlchemy column types."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.types import TypeDecorator
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
class PortableJSON(TypeDecorator):
"""Use native JSON where available, and TEXT on older MySQL servers."""
impl = JSON
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == "mysql":
return dialect.type_descriptor(Text())
return dialect.type_descriptor(JSON())
def process_bind_param(self, value: Any, dialect) -> Any:
if dialect.name != "mysql" or value is None:
return value
return json.dumps(value, ensure_ascii=False)
def process_result_value(self, value: Any, dialect) -> Any:
if dialect.name != "mysql" or value is None or not isinstance(value, str):
return value
if not value:
return None
return json.loads(value)
class PortableLongText(TypeDecorator):
"""A large text column.
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
already unbounded.
"""
impl = Text
cache_ok = True
def load_dialect_impl(self, dialect):
if dialect.name == "mysql":
return dialect.type_descriptor(LONGTEXT())
return dialect.type_descriptor(Text())
class BeijingDateTime(TypeDecorator):
"""Store timestamps as Beijing (UTC+8) wall-clock time.
The application works in UTC internally (``datetime.now(UTC)`` everywhere).
MySQL ``DATETIME`` and SQLite have no timezone, so the driver serializes the
wall-clock fields of whatever datetime it is given and drops ``tzinfo``.
Without conversion a row stores the UTC wall clock and an operator browsing
the table sees a time 8 hours behind Beijing.
This decorator converts UTC-aware values to Beijing on the way in (stored as
naive Beijing wall-clock) and re-attaches the Beijing zone on the way out, so
the database holds Beijing time while Python keeps receiving correct
timezone-aware datetimes — instant comparisons stay correct either way.
PostgreSQL has a real ``TIMESTAMP WITH TIME ZONE`` type and stores the
instant unambiguously, so values pass through unchanged there.
"""
impl = DateTime
cache_ok = True
def load_dialect_impl(self, dialect):
# timezone=True keeps PostgreSQL on timestamptz; MySQL/SQLite ignore it.
return dialect.type_descriptor(DateTime(timezone=True))
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
return value.replace(tzinfo=BEIJING_TZ)
return value.astimezone(BEIJING_TZ)
@@ -13,10 +13,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, Index, String, text
from sqlalchemy import Boolean, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class UserRow(Base):
@@ -25,7 +26,7 @@ class UserRow(Base):
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
@@ -33,7 +34,7 @@ class UserRow(Base):
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
@@ -87,6 +87,7 @@ class RunEventStore(abc.ABC):
limit: int = 50,
before_seq: int | None = None,
after_seq: int | None = None,
user_id: str | None = None,
) -> list[dict]:
"""Return displayable messages (category=message) for a specific run, ordered by seq ascending.
@@ -152,7 +152,9 @@ class JsonlRunEventStore(RunEventStore):
events = [e for e in events if e.get("event_type") in event_types]
return events[:limit]
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
# JSONL events carry no per-event owner; user_id is accepted for
# interface parity with DbRunEventStore but ignored.
events = self._read_run_events(thread_id, run_id)
filtered = [e for e in events if e.get("category") == "message"]
if before_seq is not None:
@@ -97,7 +97,9 @@ class MemoryRunEventStore(RunEventStore):
filtered = [e for e in filtered if e["event_type"] in event_types]
return filtered[:limit]
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
# The in-memory store keeps no per-event owner, so user_id is accepted
# for interface parity with DbRunEventStore but ignored.
all_events = self._events.get(thread_id, [])
filtered = [e for e in all_events if e["run_id"] == run_id and e["category"] == "message"]
if before_seq is not None:
Loaded 100 of 476 files, more files were not shown because too many files have changed in this diff. Show more