Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xhy
This commit is contained in:
commit
e2d61f4cfb
476 files changed
+103681
-9379
No files matched your search
+15
@@ -0,0 +1,15 @@
|
||||
from .graph import (
|
||||
AI_WRITING_GRAPH,
|
||||
build_ai_writing_graph,
|
||||
get_ai_writing_checkpointer,
|
||||
get_ai_writing_graph,
|
||||
shutdown_ai_writing_graph,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AI_WRITING_GRAPH",
|
||||
"build_ai_writing_graph",
|
||||
"get_ai_writing_graph",
|
||||
"get_ai_writing_checkpointer",
|
||||
"shutdown_ai_writing_graph",
|
||||
]
|
||||
+538
@@ -0,0 +1,538 @@
|
||||
"""AI 写作 LangGraph Graph,含 4 个用户干预点。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from langgraph.graph import END, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from .state import AIWritingState
|
||||
from .nodes.intent_parser import intent_parser_node
|
||||
from .nodes.researcher import researcher_node
|
||||
from .nodes.writer_outline import writer_outline_node
|
||||
from .nodes.writer_draft import writer_draft_node
|
||||
from .nodes.editor import editor_node
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 干预节点 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-1:素材确认"""
|
||||
pkg = state["material_package"]
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "material_confirm",
|
||||
"message": "素材收集专家已完成素材搜集,请确认素材后继续。",
|
||||
"materials": pkg["materials"],
|
||||
"keywords": pkg["keywords"],
|
||||
"summary": pkg["summary"],
|
||||
})
|
||||
|
||||
action = user_input.get("action", "confirm")
|
||||
|
||||
if action == "confirm":
|
||||
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
|
||||
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
|
||||
updated_pkg = {**pkg, "materials": filtered}
|
||||
return {
|
||||
"material_package": updated_pkg,
|
||||
"last_user_intervention": user_input,
|
||||
"status": "writing_outline",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材,开始规划大纲"}],
|
||||
}
|
||||
|
||||
# re_search:重新搜索
|
||||
extra_kws = user_input.get("extraKeywords") or []
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
|
||||
"status": "researching",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
|
||||
}
|
||||
|
||||
|
||||
def _normalize_outline(outline: dict) -> dict:
|
||||
"""将前端回传的 camelCase 大纲统一为后端 snake_case 格式。"""
|
||||
if not outline:
|
||||
return outline
|
||||
sections = []
|
||||
for s in outline.get("sections", []):
|
||||
sections.append({
|
||||
"section_title": s.get("section_title") or s.get("sectionTitle", ""),
|
||||
"key_points": s.get("key_points") or s.get("keyPoints", []),
|
||||
"material_ids": s.get("material_ids") or s.get("materialIds", []),
|
||||
})
|
||||
return {"title": outline.get("title", ""), "sections": sections}
|
||||
|
||||
|
||||
def _check_outline_material_support(outline: dict) -> str:
|
||||
"""轻量启发式:统计未关联任何素材的章节,返回预警文案(空串=无需预警)。
|
||||
|
||||
仅做提示、不拦截流程;真正的素材不足会在写作节点被检出并触发求助暂停。
|
||||
"""
|
||||
sections = (outline or {}).get("sections", [])
|
||||
weak = [
|
||||
(s.get("section_title") or s.get("sectionTitle") or "")
|
||||
for s in sections
|
||||
if not (s.get("material_ids") or s.get("materialIds"))
|
||||
]
|
||||
weak = [t for t in weak if t]
|
||||
if not weak:
|
||||
return ""
|
||||
return (
|
||||
f"有 {len(weak)} 个章节未关联检索素材({('、'.join(weak))})。"
|
||||
"严格依据素材模式下,这些章节在写作时可能因素材不足而暂停向你求助。"
|
||||
)
|
||||
|
||||
|
||||
async def pause_outline_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-2:大纲确认"""
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "outline_confirm",
|
||||
"message": "作家已完成大纲规划,请确认或编辑大纲后继续。",
|
||||
"outline": state["current_outline"],
|
||||
})
|
||||
|
||||
action = user_input.get("action", "confirm")
|
||||
|
||||
if action == "confirm":
|
||||
raw_outline = user_input.get("editedOutline") or state["current_outline"]
|
||||
edited = _normalize_outline(raw_outline) if raw_outline else state["current_outline"]
|
||||
events: list[dict] = [{"type": "resumed", "agent_name": "系统", "message": "用户已确认大纲,开始写作"}]
|
||||
# 阶段四:严格模式下对大纲做轻量素材充分性预警(仅提示,不拦截流程)
|
||||
if state.get("writing_mode") == "strict":
|
||||
warning = _check_outline_material_support(edited)
|
||||
if warning:
|
||||
events.append({
|
||||
"type": "notice", "agent_name": "系统",
|
||||
"title": "素材充分性提示", "detail": warning,
|
||||
})
|
||||
return {
|
||||
"current_outline": edited,
|
||||
"last_user_intervention": user_input,
|
||||
"status": "writing_draft",
|
||||
"progress_events": events,
|
||||
}
|
||||
|
||||
# re_outline
|
||||
feedback = user_input.get("outlineFeedback", "")
|
||||
updates: dict = {
|
||||
"last_user_intervention": {**user_input, "_feedback": feedback},
|
||||
"status": "writing_outline",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"重新规划大纲({feedback[:30]})"}],
|
||||
}
|
||||
# 用户在卡片里手动改过大纲 → 以手动版作为本次重规划的编辑基准
|
||||
raw_edited = user_input.get("editedOutline")
|
||||
if raw_edited:
|
||||
updates["current_outline"] = _normalize_outline(raw_edited)
|
||||
return updates
|
||||
|
||||
|
||||
async def pause_draft_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-3:草稿确认"""
|
||||
draft = state["current_draft"]
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "draft_confirm",
|
||||
"message": "草稿已完成,请确认或给出修改意见。",
|
||||
"draft_title": draft["title"],
|
||||
"revision_count": draft["revision_count"],
|
||||
"review_result": state.get("review_result"),
|
||||
})
|
||||
|
||||
action = user_input.get("action", "to_editor")
|
||||
|
||||
if action == "finalize":
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户确认定稿"}],
|
||||
}
|
||||
|
||||
if action == "user_revise":
|
||||
notes = user_input.get("userRevisionNotes", "")
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_user_notes": notes},
|
||||
"revision_count": state.get("revision_count", 0) + 1,
|
||||
"status": "revising",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": f"按用户意见重写({notes[:30]})"}],
|
||||
}
|
||||
|
||||
# to_editor
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "reviewing",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "提交编辑审核"}],
|
||||
}
|
||||
|
||||
|
||||
async def pause_section_help(state: AIWritingState) -> dict:
|
||||
"""素材不足求助点:严格模式下作家发现章节缺素材,让用户逐章决定如何处理。
|
||||
|
||||
用户对每个被阻塞章节选择:
|
||||
- supplement:补充检索素材(整体回到检索流程重走)
|
||||
- delete:删除该章节
|
||||
- loose:按通用知识续写该章节
|
||||
"""
|
||||
blocked = state.get("blocked_sections") or []
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "section_help",
|
||||
"message": "部分章节缺少足够素材支撑写作,请决定如何处理。",
|
||||
"blocked_sections": blocked,
|
||||
})
|
||||
|
||||
# sectionDecisions: [{"sectionTitle": "...", "decision": "supplement|delete|loose"}]
|
||||
by_title: dict = {}
|
||||
for d in user_input.get("sectionDecisions") or []:
|
||||
title = d.get("sectionTitle") or d.get("section_title")
|
||||
if title:
|
||||
by_title[title] = d.get("decision", "loose")
|
||||
|
||||
# 任一章节选「补充检索」→ 整体回到检索,重走「检索 → 素材确认 → 大纲 → 写作」
|
||||
supplement_reasons = [
|
||||
b["reason"] for b in blocked if by_title.get(b["section_title"]) == "supplement"
|
||||
]
|
||||
if supplement_reasons:
|
||||
return {
|
||||
# supplement_search:补充检索(保留已有素材、追加新素材),完成后直接回写作、不重排大纲
|
||||
"last_user_intervention": {**user_input, "action": "supplement_search", "_extra_keywords": supplement_reasons},
|
||||
"status": "researching",
|
||||
"blocked_sections": None,
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "补充检索素材中…"}],
|
||||
}
|
||||
|
||||
# 否则:删除标记章节、记录「通用知识续写」章节,直接回写作
|
||||
outline = state.get("current_outline") or {"title": "", "sections": []}
|
||||
kept, loose_titles = [], []
|
||||
for s in outline.get("sections", []):
|
||||
title = s.get("section_title") or s.get("sectionTitle", "")
|
||||
decision = by_title.get(title)
|
||||
if decision == "delete":
|
||||
continue
|
||||
if decision == "loose":
|
||||
loose_titles.append(title)
|
||||
kept.append(s)
|
||||
# 防御:用户把所有章节都删了 → 保留原大纲并全部按通用知识写,避免空文章
|
||||
if not kept:
|
||||
kept = outline.get("sections", [])
|
||||
loose_titles = [s.get("section_title") or s.get("sectionTitle", "") for s in kept]
|
||||
return {
|
||||
"current_outline": {**outline, "sections": kept},
|
||||
"last_user_intervention": {**user_input, "_loose_sections": loose_titles},
|
||||
"status": "writing_draft",
|
||||
"blocked_sections": None,
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "已按你的选择继续写作…"}],
|
||||
}
|
||||
|
||||
|
||||
async def pause_review_confirm(state: AIWritingState) -> dict:
|
||||
"""PAUSE-4:编辑打回确认"""
|
||||
user_input: dict = interrupt({
|
||||
"pause_point": "review_confirm",
|
||||
"message": "编辑认为文章需要修改,请决定下一步操作。",
|
||||
"review_result": state["review_result"],
|
||||
"revision_count": state.get("revision_count", 0),
|
||||
"max_revisions": state.get("max_revisions", 3),
|
||||
})
|
||||
|
||||
if user_input.get("overrideVerdict") or user_input.get("action") == "force_finalize":
|
||||
return {
|
||||
"last_user_intervention": user_input,
|
||||
"status": "done",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户强制定稿"}],
|
||||
}
|
||||
|
||||
action = user_input.get("action", "accept_review")
|
||||
extra_notes = user_input.get("userRevisionNotes", "")
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_extra_notes": extra_notes, "action": action},
|
||||
"revision_count": state.get("revision_count", 0) + 1,
|
||||
"status": "revising",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "按编辑意见重写"}],
|
||||
}
|
||||
|
||||
|
||||
# ── 条件路由 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def after_researcher(state: AIWritingState) -> str:
|
||||
"""检索完成后:补充检索(素材不足求助)直接回写作,否则去素材确认。"""
|
||||
return "writer_draft" if state.get("status") == "writing_draft" else "pause_material"
|
||||
|
||||
|
||||
def after_material_pause(state: AIWritingState) -> str:
|
||||
return "researcher" if state["status"] == "researching" else "writer_outline"
|
||||
|
||||
|
||||
def after_outline_pause(state: AIWritingState) -> str:
|
||||
return "writer_outline" if state["status"] == "writing_outline" else "writer_draft"
|
||||
|
||||
|
||||
def after_writer_draft(state: AIWritingState) -> str:
|
||||
"""写作完成后:素材不足则去求助暂停点,否则去草稿确认。"""
|
||||
return "pause_section_help" if state["status"] == "awaiting_section_help" else "pause_draft"
|
||||
|
||||
|
||||
def after_section_help_pause(state: AIWritingState) -> str:
|
||||
"""求助暂停点之后:补充检索回 researcher,否则回写作。"""
|
||||
return "researcher" if state["status"] == "researching" else "writer_draft"
|
||||
|
||||
|
||||
def after_draft_pause(state: AIWritingState) -> str:
|
||||
status = state["status"]
|
||||
if status == "done":
|
||||
return "end"
|
||||
if status == "revising":
|
||||
return "writer_draft"
|
||||
return "editor"
|
||||
|
||||
|
||||
def after_review(state: AIWritingState) -> str:
|
||||
review = state.get("review_result") or {}
|
||||
if review.get("verdict") == "pass":
|
||||
return "pause_draft"
|
||||
rev_count = state.get("revision_count", 0)
|
||||
max_rev = state.get("max_revisions", 3)
|
||||
if rev_count >= max_rev:
|
||||
return "pause_draft" # 超出上限,强制到草稿确认
|
||||
return "pause_review"
|
||||
|
||||
|
||||
def after_review_pause(state: AIWritingState) -> str:
|
||||
return "end" if state["status"] == "done" else "writer_draft"
|
||||
|
||||
|
||||
# ── 组装 Graph ────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_ai_writing_graph(checkpointer=None):
|
||||
"""构建并 compile AI 写作 graph。
|
||||
|
||||
Args:
|
||||
checkpointer: 可选的 LangGraph checkpointer。
|
||||
- ``None`` → 退回 ``MemorySaver`` —— 仅供测试 / 开发使用。
|
||||
进程或容器重启后会话状态全部丢失,前端拿着旧 session_id 调
|
||||
``/resume`` 会得到 404。
|
||||
- 其他 → 传入的 (Async)BaseCheckpointSaver 实例,用于持久化。
|
||||
"""
|
||||
g = StateGraph(AIWritingState)
|
||||
|
||||
g.add_node("intent_parser", intent_parser_node)
|
||||
g.add_node("researcher", researcher_node)
|
||||
g.add_node("pause_material", pause_material_confirm)
|
||||
g.add_node("writer_outline", writer_outline_node)
|
||||
g.add_node("pause_outline", pause_outline_confirm)
|
||||
g.add_node("writer_draft", writer_draft_node)
|
||||
g.add_node("pause_section_help", pause_section_help)
|
||||
g.add_node("pause_draft", pause_draft_confirm)
|
||||
g.add_node("editor", editor_node)
|
||||
g.add_node("pause_review", pause_review_confirm)
|
||||
|
||||
g.set_entry_point("intent_parser")
|
||||
g.add_edge("intent_parser", "researcher")
|
||||
g.add_conditional_edges("researcher", after_researcher,
|
||||
{"pause_material": "pause_material", "writer_draft": "writer_draft"})
|
||||
|
||||
g.add_conditional_edges("pause_material", after_material_pause,
|
||||
{"researcher": "researcher", "writer_outline": "writer_outline"})
|
||||
|
||||
g.add_edge("writer_outline", "pause_outline")
|
||||
g.add_conditional_edges("pause_outline", after_outline_pause,
|
||||
{"writer_outline": "writer_outline", "writer_draft": "writer_draft"})
|
||||
|
||||
g.add_conditional_edges("writer_draft", after_writer_draft,
|
||||
{"pause_section_help": "pause_section_help", "pause_draft": "pause_draft"})
|
||||
|
||||
g.add_conditional_edges("pause_section_help", after_section_help_pause,
|
||||
{"researcher": "researcher", "writer_draft": "writer_draft"})
|
||||
|
||||
g.add_conditional_edges("pause_draft", after_draft_pause,
|
||||
{"end": END, "writer_draft": "writer_draft", "editor": "editor"})
|
||||
|
||||
g.add_conditional_edges("editor", after_review,
|
||||
{"pause_draft": "pause_draft", "pause_review": "pause_review"})
|
||||
|
||||
g.add_conditional_edges("pause_review", after_review_pause,
|
||||
{"end": END, "writer_draft": "writer_draft"})
|
||||
|
||||
if checkpointer is None:
|
||||
# MemorySaver 仅作为 fallback —— 提示一下,正式部署应通过
|
||||
# get_ai_writing_graph() 拿到带持久化 checkpointer 的 graph。
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
|
||||
checkpointer = MemorySaver()
|
||||
return g.compile(checkpointer=checkpointer)
|
||||
|
||||
|
||||
# ── 持久化 checkpointer 单例(懒加载)─────────────────────────────────────────
|
||||
|
||||
# 模块级 graph 槽位。import 时为 None;首次 ``get_ai_writing_graph()`` 调用时
|
||||
# 按 config.yaml 的 checkpointer 段懒构建出真正的 graph。
|
||||
# 测试可通过 ``monkeypatch.setattr(graph_mod, "AI_WRITING_GRAPH", stub)``
|
||||
# 直接注入桩;本模块所有访问点都走 ``get_ai_writing_graph()`` 这一入口,会优先
|
||||
# 返回 module attribute 的值,保留测试桩注入路径。
|
||||
AI_WRITING_GRAPH = None
|
||||
|
||||
# 持有 async checkpointer(底层 aiosqlite Connection 或 psycopg pool 的所有者)。
|
||||
# 进程生命周期内常驻;shutdown 时由 ``shutdown_ai_writing_graph`` 显式关闭。
|
||||
_async_checkpointer_singleton = None
|
||||
|
||||
|
||||
def _fallback_to_memory_saver(reason: str):
|
||||
"""退回 InMemorySaver 的统一兜底入口。
|
||||
|
||||
把 reason 打到 warning 日志里方便运维定位。InMemorySaver 进程重启会丢
|
||||
状态,但**当次写作能跑完** —— 我们的首要目标是「写作流程能继续」,
|
||||
持久化是锦上添花。
|
||||
"""
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
logger.warning(
|
||||
"ai_writing: 持久化 checkpointer 不可用,退回 InMemorySaver。原因: %s",
|
||||
reason,
|
||||
)
|
||||
return InMemorySaver()
|
||||
|
||||
|
||||
async def _get_async_checkpointer():
|
||||
"""按 config.yaml 的 checkpointer 段返回一个 async checkpointer 实例。
|
||||
|
||||
**绝不抛异常** —— 任何失败一律退回 InMemorySaver。这是为了保证「写作
|
||||
流程能继续」这个首要目标:持久化坏了用户至少还能写完当次稿,重启会丢
|
||||
会话但不影响当下创作。
|
||||
|
||||
优先级:
|
||||
1. ``checkpointer:`` 段(``type=sqlite|postgres|memory``)
|
||||
2. fallback 到 ``InMemorySaver`` 并打 warning
|
||||
|
||||
返回的 saver 与主 agent 公用同一 sqlite 文件 —— LangGraph 的 schema
|
||||
用 ``(thread_id, checkpoint_ns)`` 做主键,AI 写作的 thread_id 是
|
||||
会话 uuid,跟主 agent 的 LangGraph thread_id 不会撞。
|
||||
"""
|
||||
global _async_checkpointer_singleton
|
||||
if _async_checkpointer_singleton is not None:
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
try:
|
||||
from deerflow.config.checkpointer_config import get_checkpointer_config
|
||||
|
||||
cp_cfg = get_checkpointer_config()
|
||||
except Exception:
|
||||
logger.exception("ai_writing: 加载 checkpointer 配置失败")
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver("加载 config 抛异常")
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg is None:
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
"config.yaml 未配置 checkpointer 段"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
_async_checkpointer_singleton = InMemorySaver()
|
||||
logger.info("ai_writing: 配置 type=memory,使用 InMemorySaver")
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "sqlite":
|
||||
try:
|
||||
import aiosqlite
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from deerflow.runtime.store._sqlite_utils import (
|
||||
ensure_sqlite_parent_dir,
|
||||
resolve_sqlite_conn_str,
|
||||
)
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(
|
||||
cp_cfg.connection_string or "ai_writing_checkpoints.db"
|
||||
)
|
||||
ensure_sqlite_parent_dir(conn_str)
|
||||
# 直接持有 aiosqlite.Connection —— 进程生命周期,shutdown 时显式关。
|
||||
conn = await aiosqlite.connect(conn_str)
|
||||
saver = AsyncSqliteSaver(conn)
|
||||
await saver.setup()
|
||||
_async_checkpointer_singleton = saver
|
||||
logger.info("ai_writing: 使用 AsyncSqliteSaver (%s)", conn_str)
|
||||
return saver
|
||||
except Exception as exc:
|
||||
# sqlite 失败原因可能是:依赖未装 / 磁盘满 / 文件权限 / 路径不存在 /
|
||||
# 文件被外部进程独占锁。任何一个都不能让 AI 写作整个不可用。
|
||||
logger.exception(
|
||||
"ai_writing: AsyncSqliteSaver 初始化失败,conn_str=%r",
|
||||
getattr(cp_cfg, "connection_string", None),
|
||||
)
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"sqlite 初始化失败: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
if cp_cfg.type == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
if not cp_cfg.connection_string:
|
||||
raise ValueError("checkpointer.connection_string 是 postgres 必填项")
|
||||
pool = AsyncConnectionPool(cp_cfg.connection_string, open=False)
|
||||
await pool.open()
|
||||
saver = AsyncPostgresSaver(pool)
|
||||
await saver.setup()
|
||||
_async_checkpointer_singleton = saver
|
||||
logger.info("ai_writing: 使用 AsyncPostgresSaver")
|
||||
return saver
|
||||
except Exception as exc:
|
||||
logger.exception("ai_writing: AsyncPostgresSaver 初始化失败")
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"postgres 初始化失败: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
# 未知 type —— config 配错也不能让服务挂
|
||||
_async_checkpointer_singleton = _fallback_to_memory_saver(
|
||||
f"未知 checkpointer.type={cp_cfg.type!r}"
|
||||
)
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
|
||||
async def get_ai_writing_graph():
|
||||
"""懒加载并返回 AI 写作 graph 单例。
|
||||
|
||||
第一次调用按 config 选定的 checkpointer 构建 graph 并缓存;后续调用直接
|
||||
返回缓存。测试可在调用前通过 ``monkeypatch.setattr`` 直接替换模块级
|
||||
``AI_WRITING_GRAPH`` —— 本函数会优先返回已设的值。
|
||||
"""
|
||||
global AI_WRITING_GRAPH
|
||||
if AI_WRITING_GRAPH is not None:
|
||||
return AI_WRITING_GRAPH
|
||||
checkpointer = await _get_async_checkpointer()
|
||||
AI_WRITING_GRAPH = build_ai_writing_graph(checkpointer=checkpointer)
|
||||
return AI_WRITING_GRAPH
|
||||
|
||||
|
||||
def get_ai_writing_checkpointer():
|
||||
"""返回当前已实例化的 checkpointer(清理任务用它来删 thread)。
|
||||
|
||||
若尚未懒构建过则返回 None。
|
||||
"""
|
||||
return _async_checkpointer_singleton
|
||||
|
||||
|
||||
async def shutdown_ai_writing_graph() -> None:
|
||||
"""关闭 saver 持有的底层连接,并清空 graph 单例。Lifespan shutdown 调用。"""
|
||||
global AI_WRITING_GRAPH, _async_checkpointer_singleton
|
||||
saver = _async_checkpointer_singleton
|
||||
_async_checkpointer_singleton = None
|
||||
AI_WRITING_GRAPH = None
|
||||
if saver is None:
|
||||
return
|
||||
# aiosqlite saver: saver.conn 是 aiosqlite.Connection
|
||||
conn = getattr(saver, "conn", None)
|
||||
if conn is not None and hasattr(conn, "close"):
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
logger.warning("ai_writing: 关闭 checkpointer connection 失败", exc_info=True)
|
||||
# postgres saver: saver.pool 是 AsyncConnectionPool
|
||||
pool = getattr(saver, "pool", None)
|
||||
if pool is not None and hasattr(pool, "close"):
|
||||
try:
|
||||
await pool.close()
|
||||
except Exception:
|
||||
logger.warning("ai_writing: 关闭 checkpointer pool 失败", exc_info=True)
|
||||
Whitespace-only changes.
+242
@@ -0,0 +1,242 @@
|
||||
"""节点共用工具:JSON 解析容错、LLM 创建。
|
||||
|
||||
弱模型经常输出不规范 JSON(中文标点、结尾多余逗号、被 markdown 包裹、
|
||||
夹带解释文字、嵌套对象)。parse_json 用多级降级策略尽量抢救出结构化结果。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_model(config: RunnableConfig, model_name: str | None = None):
|
||||
"""创建对话模型。
|
||||
|
||||
model_name 显式传入时优先使用(如素材收集专家的检索词/排序专用快模型),
|
||||
否则回退到 configurable.model_name(用户在写作表单里选的主模型)。
|
||||
"""
|
||||
from deerflow.models import create_chat_model
|
||||
name = model_name or config.get("configurable", {}).get("model_name", "") or None
|
||||
return create_chat_model(name=name)
|
||||
|
||||
|
||||
# 弱模型常把结构性标点写成中文全角符号,仅在常规解析失败后作为修复手段
|
||||
_FULLWIDTH_FIX = {
|
||||
"“": '"', # 左双引号 “
|
||||
"”": '"', # 右双引号 ”
|
||||
"‘": "'", # 左单引号 ‘
|
||||
"’": "'", # 右单引号 ’
|
||||
":": ":", # 全角冒号 :
|
||||
",": ",", # 全角逗号 ,
|
||||
"(": "(", # 全角左括号 (
|
||||
")": ")", # 全角右括号 )
|
||||
"{": "{", # 全角左花括号 {
|
||||
"}": "}", # 全角右花括号 }
|
||||
"[": "[", # 全角左方括号 [
|
||||
"]": "]", # 全角右方括号 ]
|
||||
"【": "[", # 黑方括号 【
|
||||
"】": "]", # 黑方括号 】
|
||||
}
|
||||
|
||||
|
||||
def _extract_balanced(text: str) -> str | None:
|
||||
"""从文本中提取第一个括号平衡的 {...} 块(支持嵌套,忽略字符串内的括号)。"""
|
||||
start = text.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
depth = 0
|
||||
in_str = False
|
||||
escape = False
|
||||
for i in range(start, len(text)):
|
||||
ch = text[i]
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
continue
|
||||
if ch == '"':
|
||||
in_str = not in_str
|
||||
continue
|
||||
if in_str:
|
||||
continue
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start:i + 1]
|
||||
return None
|
||||
|
||||
|
||||
def _repair_json(text: str) -> str:
|
||||
"""尽力修复弱模型常见的 JSON 错误:全角标点、结尾多余逗号。"""
|
||||
for bad, good in _FULLWIDTH_FIX.items():
|
||||
text = text.replace(bad, good)
|
||||
# 去掉对象/数组结尾多余的逗号: ,} 或 ,]
|
||||
text = re.sub(r",(\s*[}\]])", r"\1", text)
|
||||
return text
|
||||
|
||||
|
||||
def _try_load(text: str) -> dict | None:
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
return result if isinstance(result, dict) else None
|
||||
|
||||
|
||||
def parse_json(text: str, fallback: dict) -> dict:
|
||||
"""从 LLM 响应中提取 JSON 对象,多级降级,失败时返回 fallback。"""
|
||||
if not text or not text.strip():
|
||||
return fallback
|
||||
text = text.strip()
|
||||
|
||||
# 1. 直接解析
|
||||
if (r := _try_load(text)) is not None:
|
||||
return r
|
||||
|
||||
# 2. markdown 代码块内容
|
||||
m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
|
||||
if m:
|
||||
fenced = m.group(1).strip()
|
||||
if (r := _try_load(fenced)) is not None:
|
||||
return r
|
||||
text = fenced # 用代码块内容继续后续步骤
|
||||
|
||||
# 3. 平衡括号提取(处理夹带解释文字、嵌套对象的情况)
|
||||
candidate = _extract_balanced(text)
|
||||
if candidate:
|
||||
if (r := _try_load(candidate)) is not None:
|
||||
return r
|
||||
# 4. 修复后再试
|
||||
if (r := _try_load(_repair_json(candidate))) is not None:
|
||||
return r
|
||||
|
||||
# 5. 整体修复后兜底
|
||||
if (r := _try_load(_repair_json(text))) is not None:
|
||||
return r
|
||||
|
||||
logger.warning("JSON parse failed, using fallback. raw=%r", text[:200])
|
||||
return fallback
|
||||
|
||||
|
||||
def _content_to_text(content) -> str:
|
||||
"""将 LLM 响应内容统一为字符串(部分模型返回结构化 content 块列表)。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(
|
||||
part.get("text", "") if isinstance(part, dict) else str(part)
|
||||
for part in content
|
||||
)
|
||||
return str(content)
|
||||
|
||||
|
||||
# 中文常见标点:参与「字数」统计,与 Word 字数统计 / 用户直觉口径对齐
|
||||
_CJK_PUNCTUATION = ",。、;:?!“”‘’()《》【】「」『』〈〉—…·~"
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
"""统计中文语境下的「字数」。
|
||||
|
||||
口径:汉字逐字计 + 中文标点逐个计 + 连续数字/英文各算 1 字。
|
||||
不计入空白、markdown 语法符号(# * - 等)与 ASCII 标点。
|
||||
前后端共用同一口径(前端 utils/wordCount.ts 实现一致)。
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
cjk = len(re.findall(r"[一-鿿]", text))
|
||||
cjk_punct = sum(1 for c in text if c in _CJK_PUNCTUATION)
|
||||
alnum_runs = len(re.findall(r"[A-Za-z0-9]+", text))
|
||||
return cjk + cjk_punct + alnum_runs
|
||||
|
||||
|
||||
class ThinkTagParser:
|
||||
"""流式 <think>...</think> 标签解析器。
|
||||
|
||||
模型可能在任意 chunk 边界截断标签,通过尾部缓冲处理跨 chunk 边界情况。
|
||||
调用 feed(chunk) 返回 list[tuple[bool, str]],每项为 (is_thinking, text)。
|
||||
"""
|
||||
|
||||
_OPEN_TAG = "<think>"
|
||||
_CLOSE_TAG = "</think>"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._in_think = False
|
||||
self._buf = "" # 潜在标签的部分匹配缓冲
|
||||
|
||||
def feed(self, chunk: str) -> list[tuple[bool, str]]:
|
||||
"""处理一个流式 chunk,返回 (is_thinking, text) 片段列表。"""
|
||||
self._buf += chunk
|
||||
result: list[tuple[bool, str]] = []
|
||||
|
||||
while self._buf:
|
||||
if self._in_think:
|
||||
tag = self._CLOSE_TAG
|
||||
idx = self._buf.find(tag)
|
||||
if idx != -1:
|
||||
if idx > 0:
|
||||
result.append((True, self._buf[:idx]))
|
||||
self._buf = self._buf[idx + len(tag):]
|
||||
self._in_think = False
|
||||
else:
|
||||
partial = self._partial_suffix(self._buf, tag)
|
||||
if partial:
|
||||
result.append((True, self._buf[:-partial]))
|
||||
self._buf = self._buf[-partial:]
|
||||
else:
|
||||
result.append((True, self._buf))
|
||||
self._buf = ""
|
||||
break
|
||||
else:
|
||||
tag = self._OPEN_TAG
|
||||
idx = self._buf.find(tag)
|
||||
if idx != -1:
|
||||
if idx > 0:
|
||||
result.append((False, self._buf[:idx]))
|
||||
self._buf = self._buf[idx + len(tag):]
|
||||
self._in_think = True
|
||||
else:
|
||||
partial = self._partial_suffix(self._buf, tag)
|
||||
if partial:
|
||||
result.append((False, self._buf[:-partial]))
|
||||
self._buf = self._buf[-partial:]
|
||||
else:
|
||||
result.append((False, self._buf))
|
||||
self._buf = ""
|
||||
break
|
||||
|
||||
return [(is_t, t) for (is_t, t) in result if t]
|
||||
|
||||
@staticmethod
|
||||
def _partial_suffix(text: str, tag: str) -> int:
|
||||
"""返回 text 尾部与 tag 前缀匹配的最长长度(>0 说明可能是部分标签)。"""
|
||||
max_check = min(len(tag) - 1, len(text))
|
||||
for length in range(max_check, 0, -1):
|
||||
if tag.startswith(text[-length:]):
|
||||
return length
|
||||
return 0
|
||||
|
||||
|
||||
async def llm_json(system: str, user: str, fallback: dict, config: RunnableConfig, model_name: str | None = None) -> dict:
|
||||
"""调用 LLM,解析 JSON 输出,失败用 fallback。
|
||||
|
||||
model_name 可覆盖默认模型(如检索词/排序使用配置的专用快模型)。
|
||||
"""
|
||||
messages = [SystemMessage(content=system), HumanMessage(content=user)]
|
||||
try:
|
||||
# get_model 放进 try:模型名在 config.yaml 找不到时 create_chat_model 会抛
|
||||
# ValueError,若留在 try 外会逐层上抛使整个图硬崩溃。
|
||||
model = get_model(config, model_name=model_name)
|
||||
resp = await model.ainvoke(messages)
|
||||
return parse_json(_content_to_text(resp.content), fallback)
|
||||
except Exception as e:
|
||||
logger.error("llm_json failed: %s", e, exc_info=True)
|
||||
return {**fallback, "_llm_error": str(e)}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
"""编辑节点:三项独立 LLM 审查(事实/逻辑/语言),逐 token 流式推送以支持打字机效果。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, ReviewResult, ReviewIssue
|
||||
from ..prompts.editor_prompts import (
|
||||
EDITOR_SYSTEM, FACT_REVIEW_USER, LOGIC_REVIEW_USER, LANGUAGE_REVIEW_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_PASS_THRESHOLD = 75
|
||||
|
||||
|
||||
def _materials_summary(pkg: dict) -> str:
|
||||
lines = []
|
||||
for m in (pkg.get("materials") or [])[:6]:
|
||||
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:200]}")
|
||||
return "\n".join(lines) or "(无素材)"
|
||||
|
||||
|
||||
def _parse_issues(raw_issues: list, category: str) -> list[ReviewIssue]:
|
||||
issues = []
|
||||
for item in (raw_issues or []):
|
||||
issues.append(ReviewIssue(
|
||||
category=category,
|
||||
severity=item.get("severity", "minor"),
|
||||
location=item.get("location", ""),
|
||||
issue=item.get("issue", ""),
|
||||
suggestion=item.get("suggestion", ""),
|
||||
))
|
||||
return issues
|
||||
|
||||
|
||||
async def _stream_one_review(
|
||||
model, user_prompt: str, category: str,
|
||||
sse_queue: asyncio.Queue | None, session_id: str,
|
||||
) -> str:
|
||||
"""流式执行单项审查,逐 chunk 推送 review_chunk 事件,返回累积的正文文本(用于 JSON 解析)。"""
|
||||
full_text = "" # 仅累积正文(非思考),供后续 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=EDITOR_SYSTEM),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("review_chunk", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
return full_text
|
||||
|
||||
|
||||
async def editor_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
draft = state.get("current_draft")
|
||||
if not draft:
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{"type": "error", "agent_name": "编辑", "message": "草稿为空,无法审核"}],
|
||||
}
|
||||
|
||||
article = draft["full_markdown"]
|
||||
pkg = state.get("material_package") or {}
|
||||
mats = _materials_summary(pkg)
|
||||
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
pass_threshold = int(ai_writing_config.get("pass_threshold", _DEFAULT_PASS_THRESHOLD))
|
||||
|
||||
REVIEW_TASKS = [
|
||||
("fact", FACT_REVIEW_USER.format(materials_summary=mats, article=article)),
|
||||
("logic", LOGIC_REVIEW_USER.format(article=article)),
|
||||
("language", LANGUAGE_REVIEW_USER.format(article=article)),
|
||||
]
|
||||
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
except Exception as e:
|
||||
logger.error("editor model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error", "agent_name": "编辑",
|
||||
"title": "模型错误", "detail": str(e), "message": "审核失败",
|
||||
}],
|
||||
}
|
||||
results: dict[str, dict] = {}
|
||||
review_errors: list[str] = [] # 分项审查的模型异常,供完成后非致命提示
|
||||
|
||||
_CATEGORY_LABEL = {"fact": "事实性", "logic": "逻辑性", "language": "语言文字"}
|
||||
|
||||
for category, user_prompt in REVIEW_TASKS:
|
||||
fallback = {"score": 80, "issues": []}
|
||||
try:
|
||||
full_text = await _stream_one_review(model, user_prompt, category, sse_queue, session_id)
|
||||
res = parse_json(full_text, fallback)
|
||||
except Exception as e:
|
||||
logger.warning("editor review [%s] failed: %s", category, e)
|
||||
res = fallback
|
||||
review_errors.append(f"「{_CATEGORY_LABEL.get(category, category)}审查」{type(e).__name__}: {e}")
|
||||
|
||||
results[category] = res
|
||||
|
||||
# 这一项审查完成 —— 推送结构化结果(前端据此把流式文本替换为评分/意见)
|
||||
if sse_queue is not None:
|
||||
issues = _parse_issues(res.get("issues", []), category)
|
||||
await sse_queue.put(("review_item_done", {
|
||||
"session_id": session_id,
|
||||
"category": category,
|
||||
"score": int(res.get("score", 80)),
|
||||
"issues": [dict(iss) for iss in issues],
|
||||
}))
|
||||
else:
|
||||
logger.debug("editor [%s] done (no sse_queue)", category)
|
||||
|
||||
# 汇总
|
||||
fact_score = int(results["fact"].get("score", 80))
|
||||
logic_score = int(results["logic"].get("score", 80))
|
||||
lang_score = int(results["language"].get("score", 80))
|
||||
overall_score = (fact_score + logic_score + lang_score) // 3
|
||||
|
||||
all_issues: list[ReviewIssue] = (
|
||||
_parse_issues(results["fact"].get("issues", []), "fact") +
|
||||
_parse_issues(results["logic"].get("issues", []), "logic") +
|
||||
_parse_issues(results["language"].get("issues", []), "language")
|
||||
)
|
||||
all_issues.sort(key=lambda x: (0 if x["severity"] == "major" else 1))
|
||||
|
||||
verdict = "pass" if overall_score >= pass_threshold else "reject"
|
||||
|
||||
review = ReviewResult(
|
||||
verdict=verdict,
|
||||
fact_score=fact_score,
|
||||
logic_score=logic_score,
|
||||
language_score=lang_score,
|
||||
overall_score=overall_score,
|
||||
pass_threshold=pass_threshold,
|
||||
revision_notes=all_issues,
|
||||
)
|
||||
|
||||
# 分项审查出现模型异常时,以非致命 notice 提示用户,审核结论仍照常产出
|
||||
notices: list[dict] = []
|
||||
if review_errors:
|
||||
notices.append({
|
||||
"type": "notice",
|
||||
"agent_name": "编辑",
|
||||
"title": "模型错误",
|
||||
"detail": "部分审查项调用失败,已按默认分兜底。原因:" + ";".join(review_errors[:3]),
|
||||
})
|
||||
|
||||
return {
|
||||
"review_result": review,
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "编辑", "message": "开始审核"},
|
||||
*notices,
|
||||
{
|
||||
"type": "review_ready",
|
||||
"agent_name": "编辑",
|
||||
"message": f"审核完成,综合得分 {overall_score},结论:{'通过' if verdict == 'pass' else '打回修改'}",
|
||||
},
|
||||
],
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""意图解析节点:将用户自然语言意图结构化为写作配置。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState
|
||||
from ._utils import llm_json
|
||||
|
||||
# 兜底类型列表,当 config 中未注入 valid_article_types 时使用
|
||||
_DEFAULT_ARTICLE_TYPES: list[dict] = [
|
||||
{"key": "news", "label": "新闻资讯"},
|
||||
{"key": "analysis", "label": "分析报道"},
|
||||
{"key": "feature", "label": "深度特稿"},
|
||||
{"key": "report", "label": "调研报告"},
|
||||
]
|
||||
|
||||
_SYSTEM_TEMPLATE = """你是一名写作助手。
|
||||
你的任务:分析用户的写作意图,提取结构化的写作配置。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- article_type 只能是以下之一:{type_options}。
|
||||
|
||||
【完整示例】
|
||||
输入意图:帮我写一篇面向行业专家的深度文章,分析今年新能源汽车出口为什么大涨,大概两千字
|
||||
正确输出:
|
||||
{{"article_type": "feature", "target_audience": "行业专家", "word_count_target": 2000, "refined_intent": "分析今年新能源汽车出口高速增长的原因"}}"""
|
||||
|
||||
_USER_TEMPLATE = """用户写作意图:{user_intent}
|
||||
|
||||
请分析并输出以下 JSON(如果意图中未明确某字段,给出合理默认值):
|
||||
{{{{
|
||||
"article_type": "{type_options} 之一",
|
||||
"target_audience": "目标读者描述",
|
||||
"word_count_target": 800,
|
||||
"refined_intent": "精炼后的写作意图(一句话)"
|
||||
}}}}"""
|
||||
|
||||
|
||||
def _build_type_options(types: list[dict]) -> str:
|
||||
"""将类型列表格式化为提示词中的枚举说明,如 news(新闻资讯)/ analysis(分析报道)。"""
|
||||
return " / ".join(f'{t["key"]}({t["label"]})' for t in types)
|
||||
|
||||
|
||||
def _coerce_word_count(raw) -> int | None:
|
||||
"""稳健地把模型返回的字数字段转成正整数;无法解析时返回 None。
|
||||
|
||||
弱模型(尤其老模型被陌生词带偏时)常返回「约800字」「两千」「800.0」
|
||||
这类非纯数字形式,直接 int() 会抛 ValueError/TypeError 使整个图崩溃。
|
||||
此处用「提取首个数字串」策略容错,解析不出就返回 None 让调用方保留默认值。
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, bool): # bool 是 int 子类,需先排除
|
||||
return None
|
||||
if isinstance(raw, (int, float)):
|
||||
value = int(raw)
|
||||
return value if value > 0 else None
|
||||
if not isinstance(raw, str):
|
||||
# dict / list 等结构化类型:str() 化后可能误含数字,直接判为无法解析
|
||||
return None
|
||||
m = re.search(r"\d[\d,]*", raw)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
value = int(m.group().replace(",", ""))
|
||||
except ValueError:
|
||||
return None
|
||||
return value if value > 0 else None
|
||||
|
||||
|
||||
async def intent_parser_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
# 从 configurable 读取动态类型列表,未配置时退回默认值
|
||||
configurable = (config or {}).get("configurable", {})
|
||||
valid_article_types: list[dict] = configurable.get("valid_article_types") or _DEFAULT_ARTICLE_TYPES
|
||||
valid_keys: list[str] = [t["key"] for t in valid_article_types]
|
||||
default_key: str = valid_keys[0] if valid_keys else "analysis"
|
||||
|
||||
type_options = _build_type_options(valid_article_types)
|
||||
system = _SYSTEM_TEMPLATE.format(type_options=type_options)
|
||||
user = _USER_TEMPLATE.format(
|
||||
user_intent=state["user_intent"],
|
||||
type_options=" | ".join(valid_keys),
|
||||
)
|
||||
|
||||
fallback_type = state.get("article_type", default_key)
|
||||
if fallback_type not in valid_keys:
|
||||
fallback_type = default_key
|
||||
|
||||
result = await llm_json(
|
||||
system,
|
||||
user,
|
||||
fallback={
|
||||
"article_type": fallback_type,
|
||||
"target_audience": state.get("target_audience", "普通读者"),
|
||||
"word_count_target": state.get("word_count_target", 800),
|
||||
"refined_intent": state["user_intent"],
|
||||
},
|
||||
config=config,
|
||||
)
|
||||
|
||||
# 意图解析模型调用失败时以非致命 notice 提示用户(llm_json 已用 fallback 兜底,
|
||||
# 流程仍可继续,但用户需知道结构化配置可能不准)。
|
||||
progress_events: list[dict] = []
|
||||
if result.get("_llm_error"):
|
||||
progress_events.append({
|
||||
"type": "notice",
|
||||
"agent_name": "系统",
|
||||
"title": "模型错误",
|
||||
"detail": f"意图解析失败,已按默认配置兜底。原因:{result['_llm_error']}",
|
||||
})
|
||||
progress_events.append({"type": "step_done", "agent_name": "系统", "message": "意图解析完成"})
|
||||
|
||||
updates: dict = {"status": "researching", "progress_events": progress_events}
|
||||
|
||||
# 校验 LLM 返回的 article_type 是否在有效范围内,不合法时回退到默认值
|
||||
raw_type = result.get("article_type", "")
|
||||
updates["article_type"] = raw_type if raw_type in valid_keys else fallback_type
|
||||
|
||||
if result.get("target_audience"):
|
||||
updates["target_audience"] = result["target_audience"]
|
||||
word_count = _coerce_word_count(result.get("word_count_target"))
|
||||
if word_count is not None:
|
||||
updates["word_count_target"] = word_count
|
||||
if result.get("refined_intent"):
|
||||
updates["user_intent"] = result["refined_intent"]
|
||||
|
||||
return updates
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
"""素材收集专家节点:关键词提取 → 并发搜索 → 相关度排序。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..search import get_search_provider, SearchResult
|
||||
from ..state import AIWritingState, Material, MaterialPackage
|
||||
from ..prompts.researcher_prompts import (
|
||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
||||
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
|
||||
)
|
||||
from ._utils import llm_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 素材总量上限边界:config.yaml 的 ai_writing.max_materials,非法值回退默认 20
|
||||
_DEFAULT_MAX_MATERIALS = 20
|
||||
_MIN_MAX_MATERIALS = 1
|
||||
_MAX_MAX_MATERIALS = 100
|
||||
|
||||
# 素材检索关键词数量边界:用户可指定 1~10 个,非法值回退默认 6
|
||||
_DEFAULT_KEYWORD_COUNT = 6
|
||||
_MAX_KEYWORD_COUNT = 10
|
||||
|
||||
|
||||
def _resolve_keyword_count(raw: object) -> int:
|
||||
"""把用户传入的检索关键词数量规整到 [1, _MAX_KEYWORD_COUNT],非法值回退默认 6。"""
|
||||
try:
|
||||
n = int(raw) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_KEYWORD_COUNT
|
||||
return max(1, min(_MAX_KEYWORD_COUNT, n))
|
||||
|
||||
|
||||
def _resolve_max_materials(raw: object) -> int:
|
||||
"""把 config.yaml 配置的素材总量上限规整到 [1, 100],非法值回退默认 20。"""
|
||||
try:
|
||||
n = int(raw) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_MAX_MATERIALS
|
||||
return max(_MIN_MAX_MATERIALS, min(_MAX_MAX_MATERIALS, n))
|
||||
|
||||
|
||||
# 占位/示意性关键词(模型误抄提示词模板时产生),检索前剔除
|
||||
_PLACEHOLDER_KEYWORD_RE = re.compile(r"^(关键词\d*|keyword\d*|kw\d*|\.+|…+)$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _clean_keywords(raw: object, fallback: str) -> list[str]:
|
||||
"""清洗 LLM 产出的关键词:去空白、丢弃占位符(如「关键词1」「...」)、去重。
|
||||
|
||||
全部被滤掉时回退为写作意图本身,避免拿占位词检索出一堆无关素材。
|
||||
"""
|
||||
cleaned: list[str] = []
|
||||
for kw in raw if isinstance(raw, list) else []:
|
||||
s = str(kw).strip()
|
||||
if not s or _PLACEHOLDER_KEYWORD_RE.match(s):
|
||||
continue
|
||||
cleaned.append(s)
|
||||
deduped = list(dict.fromkeys(cleaned))
|
||||
return deduped or [fallback]
|
||||
|
||||
|
||||
def _per_keyword_limit(max_materials: int, total_keywords: int) -> int:
|
||||
"""把素材总量上限均分到每个检索词上的检索条数(向上取整)。
|
||||
|
||||
向上取整保证合并去重后素材数仍能接近总量上限;最终会再统一截断到上限。
|
||||
total_keywords <= 0 时回退为总量上限本身。
|
||||
"""
|
||||
if total_keywords <= 0:
|
||||
return max_materials
|
||||
return (max_materials + total_keywords - 1) // total_keywords
|
||||
|
||||
|
||||
def _notice(title: str, detail: str, raw: str = "") -> dict:
|
||||
"""构造一个非致命告警事件,前端在时间线显示「标题 + 真实原因」,不打断写作流程。
|
||||
|
||||
raw 非空时附带接口的原始错误响应体,前端默认折叠展示。
|
||||
"""
|
||||
event = {"type": "notice", "agent_name": "素材收集专家", "title": title, "detail": detail}
|
||||
if raw:
|
||||
event["raw"] = raw
|
||||
return event
|
||||
|
||||
|
||||
async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
intent = state["user_intent"]
|
||||
article_type = state.get("article_type", "analysis")
|
||||
target_audience = state.get("target_audience", "普通读者")
|
||||
|
||||
# 检查是否是补充搜索(PAUSE-1 re_search action)
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
|
||||
|
||||
# 素材不足求助点触发的「补充检索」:保留原大纲,检索完直接回写作、不走素材确认/大纲
|
||||
is_supplement = intervention.get("action") == "supplement_search"
|
||||
|
||||
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
|
||||
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
|
||||
|
||||
# SSE 实时进度推送(与 outline/draft 节点共用 sse_queue 机制)
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue = cfg.get("sse_queue")
|
||||
sse_session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
# 素材收集专家专用模型:检索词生成 / 素材排序 可在 config.yaml 的
|
||||
# ai_writing 段单独配置(keyword_model / rank_model),留空则回退写作主模型。
|
||||
ai_writing_config = cfg.get("ai_writing_config", {}) or {}
|
||||
keyword_model: str | None = ai_writing_config.get("keyword_model") or None
|
||||
rank_model: str | None = ai_writing_config.get("rank_model") or None
|
||||
# 素材总量上限(config.yaml: ai_writing.max_materials),按检索词数均分到每个词
|
||||
max_materials = _resolve_max_materials(ai_writing_config.get("max_materials"))
|
||||
|
||||
async def _emit(message: str) -> None:
|
||||
if sse_queue is None:
|
||||
return
|
||||
await sse_queue.put(("research_progress", {
|
||||
"session_id": sse_session_id,
|
||||
"message": message,
|
||||
}))
|
||||
|
||||
# Step 1: 提取关键词
|
||||
await _emit("正在生成检索词…")
|
||||
kw_result = await llm_json(
|
||||
EXTRACT_KEYWORDS_SYSTEM,
|
||||
EXTRACT_KEYWORDS_USER.format(
|
||||
user_intent=intent,
|
||||
article_type=article_type,
|
||||
target_audience=target_audience,
|
||||
keyword_count=keyword_count,
|
||||
),
|
||||
fallback={"keywords": [intent]},
|
||||
config=config,
|
||||
model_name=keyword_model,
|
||||
)
|
||||
# 清洗关键词:剔除占位符 / 空值,全空时回退写作意图
|
||||
keywords: list[str] = _clean_keywords(kw_result.get("keywords"), intent)
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 2: 并发搜索
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
# 按用户指定的关键词数量截断;补充搜索时不截断用户额外添加的关键词
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
await _emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
|
||||
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
|
||||
completed = 0
|
||||
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
nonlocal completed
|
||||
try:
|
||||
return await provider.search(kw, max_results=per_keyword)
|
||||
finally:
|
||||
completed += 1
|
||||
if completed < total:
|
||||
await _emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
else:
|
||||
await _emit(f"已完成全部 {total} 次检索")
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
|
||||
# 合并去重
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials: list[Material] = []
|
||||
existing_pkg = state.get("material_package")
|
||||
if existing_pkg and intervention.get("action") != "re_search":
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
|
||||
search_errors: list[str] = []
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
for r in results:
|
||||
if r.url and r.url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(r.url)
|
||||
raw_materials.append(Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
|
||||
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
|
||||
await _emit("检索完成,正在筛选排序素材…")
|
||||
rank_error = ""
|
||||
if len(raw_materials) > max_materials:
|
||||
materials_list = "\n".join(
|
||||
f"- ID: {m['id']} 标题: {m['title']} 摘要: {m['content'][:100]}"
|
||||
for m in raw_materials
|
||||
)
|
||||
rank_result = await llm_json(
|
||||
RANK_MATERIALS_SYSTEM,
|
||||
RANK_MATERIALS_USER.format(user_intent=intent, materials_list=materials_list),
|
||||
fallback={"ranked_ids": [m["id"] for m in raw_materials[:max_materials]], "summary": f"共搜索到{len(raw_materials)}条素材"},
|
||||
config=config,
|
||||
model_name=rank_model,
|
||||
)
|
||||
rank_error = rank_result.get("_llm_error", "")
|
||||
ranked_ids: list[str] = rank_result.get("ranked_ids", [m["id"] for m in raw_materials])
|
||||
id_to_mat = {m["id"]: m for m in raw_materials}
|
||||
sorted_materials = [id_to_mat[mid] for mid in ranked_ids if mid in id_to_mat]
|
||||
# 加入未被排序的素材
|
||||
sorted_ids_set = set(ranked_ids)
|
||||
for m in raw_materials:
|
||||
if m["id"] not in sorted_ids_set:
|
||||
sorted_materials.append(m)
|
||||
materials = sorted_materials[:max_materials]
|
||||
summary = rank_result.get("summary", f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条")
|
||||
else:
|
||||
materials = sorted(raw_materials, key=lambda m: m.get("relevance_score", 0), reverse=True)[:max_materials]
|
||||
summary = f"共搜索到{len(raw_materials)}条素材,精选{len(materials)}条"
|
||||
|
||||
# 空结果兜底:未检索到任何素材时给出明确文案,前端据此弹出「无数据继续」选项
|
||||
if not materials:
|
||||
summary = "未检索到相关素材,可重新搜索或在无素材的情况下继续写作"
|
||||
|
||||
pkg = MaterialPackage(keywords=keywords, materials=materials, summary=summary)
|
||||
|
||||
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
|
||||
notices: list[dict] = []
|
||||
if kw_result.get("_llm_error"):
|
||||
notices.append(_notice("模型错误", f"检索词生成失败,已用默认关键词兜底。原因:{kw_result['_llm_error']}"))
|
||||
if rank_error:
|
||||
notices.append(_notice("模型错误", f"素材排序失败,已按相关度兜底排序。原因:{rank_error}"))
|
||||
if search_errors:
|
||||
uniq = list(dict.fromkeys(search_errors))
|
||||
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
|
||||
notices.append(_notice(
|
||||
"ES 波动导致检索异常",
|
||||
";".join(uniq[:3]),
|
||||
raw="\n\n----------\n\n".join(raw_dumps[:3]),
|
||||
))
|
||||
|
||||
return {
|
||||
"material_package": pkg,
|
||||
# 补充检索:直接回写作(沿用原大纲);常规检索:进入素材确认暂停点
|
||||
"status": "writing_draft" if is_supplement else "awaiting_material_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "素材收集专家", "message": f"搜索关键词:{', '.join(keywords[:3])}"},
|
||||
*notices,
|
||||
{"type": "materials_ready", "agent_name": "素材收集专家", "message": summary},
|
||||
],
|
||||
}
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
"""作家节点 Phase 2:按大纲逐段写正文,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, DraftArticle
|
||||
from ..prompts.writer_prompts import (
|
||||
DRAFT_SECTION_SYSTEM, DRAFT_SECTION_USER, DRAFT_EXPAND_USER,
|
||||
WRITER_MODE_LOOSE, WRITER_MODE_STRICT, SECTION_BLOCKED_MARKER,
|
||||
)
|
||||
from ._utils import get_model, _content_to_text, count_words, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 章节正文里的 Markdown 标题行(允许前导空白)
|
||||
_HEADING_PREFIX_RE = re.compile(r"^[ \t]*#{1,6}[ \t]+", re.MULTILINE)
|
||||
|
||||
|
||||
def _normalize_section_headings(text: str) -> str:
|
||||
"""把章节正文里所有 Markdown 标题行统一压成三级标题(###)。
|
||||
|
||||
作家被要求只用 ### 作为「要点」小标题;模型偶尔误用 #/##/#### 时在此纠正,
|
||||
保证文档标题层级符合「文章 # → 章节 ## → 要点 ###」,
|
||||
前端目录栏据此才能把要点正确识别为二级标题。
|
||||
"""
|
||||
return _HEADING_PREFIX_RE.sub("### ", text)
|
||||
|
||||
|
||||
# 疑似「拒绝 / 元话语」的强特征短语:模型把对用户说的话当成正文输出时出现,
|
||||
# 这些短语几乎不可能出现在正常文章里,用作安全网识别。
|
||||
_REFUSAL_MARKERS = (
|
||||
"我无法完成", "无法完成此任务", "无法完成该任务", "无法完成这个任务",
|
||||
"无法完成本次", "无法写出符合", "无法撰写这篇", "无法基于现有素材",
|
||||
"无法基于当前素材", "无法基于提供的素材",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_refusal(text: str) -> bool:
|
||||
"""判断一段章节产出是否疑似「拒绝 / 元话语」而非真正的文章正文。
|
||||
|
||||
模型遇到素材与意图不匹配时,可能把「我无法完成此任务」之类对用户说的话
|
||||
当成正文输出。这里用强特征短语做启发式识别(安全网,非强校验)。
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return False
|
||||
return any(marker in text for marker in _REFUSAL_MARKERS)
|
||||
|
||||
|
||||
def _get_section_materials(material_ids: list[str], pkg: dict) -> str:
|
||||
mat_map = {m["id"]: m for m in pkg.get("materials", [])}
|
||||
lines = []
|
||||
for mid in material_ids:
|
||||
if mid in mat_map:
|
||||
m = mat_map[mid]
|
||||
lines.append(f"[{mid}] {m['title']}: {m['content'][:300]}")
|
||||
if not lines:
|
||||
for m in pkg.get("materials", [])[:2]:
|
||||
lines.append(f"[{m['id']}] {m['title']}: {m['content'][:300]}")
|
||||
return "\n".join(lines) if lines else "(无相关素材)"
|
||||
|
||||
|
||||
async def writer_draft_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
outline = state.get("current_outline")
|
||||
if not outline:
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{"type": "error", "agent_name": "作家", "message": "大纲为空,无法写作"}],
|
||||
}
|
||||
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
revision_count = state.get("revision_count", 0)
|
||||
|
||||
# 写作模式;用户在「素材不足求助」里选「按通用知识续写」的章节强制走宽松
|
||||
writing_mode = state.get("writing_mode", "loose")
|
||||
loose_titles: set[str] = set(intervention.get("_loose_sections") or [])
|
||||
|
||||
# SSE 流式推送所需
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
# 修改意见(来自用户或编辑)
|
||||
revision_notes = ""
|
||||
if intervention.get("action") == "user_revise":
|
||||
notes = intervention.get("_user_notes") or intervention.get("user_revision_notes", "")
|
||||
if notes:
|
||||
revision_notes = f"\n用户修改意见:{notes}"
|
||||
elif intervention.get("action") in ("accept_review", "custom_revise"):
|
||||
review = state.get("review_result") or {}
|
||||
issues = review.get("revision_notes", [])
|
||||
extra = intervention.get("_extra_notes") or intervention.get("user_revision_notes", "")
|
||||
parts = []
|
||||
for iss in issues[:5]:
|
||||
parts.append(f"- [{iss.get('severity','minor')}] {iss.get('location','')}: {iss.get('suggestion','')}")
|
||||
if extra:
|
||||
parts.append(f"- 用户补充:{extra}")
|
||||
if parts:
|
||||
revision_notes = "\n修改要求:\n" + "\n".join(parts)
|
||||
|
||||
word_count_target = state.get("word_count_target", 800)
|
||||
sections = outline.get("sections", [])
|
||||
n_sections = max(len(sections), 1)
|
||||
# 单节硬下限:总目标 / 节数,乘 1.1 过量系数补偿模型对短目标的天然欠量;单节至少 150 字
|
||||
words_per_section = max(150, int(word_count_target * 1.1 / n_sections))
|
||||
article_title = outline.get("title", "文章")
|
||||
|
||||
# 兼容后端 snake_case 与前端编辑后回传的 camelCase
|
||||
def _section_title(idx: int, sec: dict) -> str:
|
||||
return sec.get("section_title") or sec.get("sectionTitle") or f"章节{idx + 1}"
|
||||
|
||||
# 模型创建守卫:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 不拦截会硬崩溃整个图;此处转成可恢复的 status:error。
|
||||
try:
|
||||
model = get_model(config)
|
||||
except Exception as e:
|
||||
logger.error("draft model creation failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error", "agent_name": "作家",
|
||||
"title": "模型错误", "detail": str(e), "message": "草稿生成失败",
|
||||
}],
|
||||
}
|
||||
section_titles: list[str] = []
|
||||
section_bufs: list[str] = [] # 各章节正文(不含 ## 标题),供扩写兜底逐节替换
|
||||
section_errors: list[str] = [] # 分节写作的模型异常,供完成后非致命提示
|
||||
blocked_sections: list[dict] = [] # 严格模式下素材不足、无法写作的章节
|
||||
|
||||
for i, section in enumerate(sections):
|
||||
sec_title = _section_title(i, section)
|
||||
key_points = section.get("key_points") or section.get("keyPoints") or []
|
||||
mat_ids = section.get("material_ids") or section.get("materialIds") or []
|
||||
|
||||
section_materials = _get_section_materials(mat_ids, pkg)
|
||||
key_points_str = "\n".join(f"- {p}" for p in key_points)
|
||||
|
||||
# 文章整体结构:让作家看到全文章节与自己的定位,避免章节间内容重复、引言结论脱节
|
||||
outline_context = "\n".join(
|
||||
f"{j + 1}. {_section_title(j, s)}" + (" ← 当前正在写这一章" if j == i else "")
|
||||
for j, s in enumerate(sections)
|
||||
)
|
||||
|
||||
# 本章节是否走严格模式(全局严格 且 未被用户标记为「通用知识续写」)
|
||||
section_is_strict = writing_mode == "strict" and sec_title not in loose_titles
|
||||
mode_section = WRITER_MODE_STRICT if section_is_strict else WRITER_MODE_LOOSE
|
||||
|
||||
user_msg = DRAFT_SECTION_USER.format(
|
||||
article_title=article_title,
|
||||
outline_context=outline_context,
|
||||
section_title=sec_title,
|
||||
key_points=key_points_str,
|
||||
section_materials=section_materials,
|
||||
revision_section=revision_notes,
|
||||
mode_section=mode_section,
|
||||
target_words=words_per_section,
|
||||
)
|
||||
|
||||
section_buf = "" # 只累积正文(非思考)用于拼接 Markdown
|
||||
parser = ThinkTagParser()
|
||||
try:
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=DRAFT_SECTION_SYSTEM),
|
||||
HumanMessage(content=user_msg),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
section_buf += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"is_section_done": False,
|
||||
}))
|
||||
|
||||
# 章节完成
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("draft_chunk", {
|
||||
"session_id": session_id,
|
||||
"section_index": i,
|
||||
"section_title": sec_title,
|
||||
"text": "",
|
||||
"is_section_done": True,
|
||||
}))
|
||||
|
||||
cleaned_buf = section_buf.strip()
|
||||
blocked_reason = ""
|
||||
# 严格模式:作家用哨兵声明素材不足,或仍输出了拒绝话术 → 标记为待求助章节
|
||||
if section_is_strict and SECTION_BLOCKED_MARKER in cleaned_buf:
|
||||
blocked_reason = (
|
||||
cleaned_buf.split(SECTION_BLOCKED_MARKER, 1)[1].strip()[:120]
|
||||
or "素材不足以支撑本章节写作"
|
||||
)
|
||||
elif _looks_like_refusal(cleaned_buf):
|
||||
if section_is_strict:
|
||||
blocked_reason = "素材不足以支撑本章节写作"
|
||||
else:
|
||||
# 宽松模式:模型仍拒绝,安全网占位,不打断流程
|
||||
logger.warning("draft section [%s] looks like a refusal, replaced with placeholder", sec_title)
|
||||
section_errors.append(f"「{sec_title}」生成内容疑似拒绝/无关,已占位")
|
||||
cleaned_buf = "(本节内容生成异常,建议重新生成或补充相关素材)"
|
||||
if blocked_reason:
|
||||
blocked_sections.append({"section_title": sec_title, "reason": blocked_reason})
|
||||
cleaned_buf = f"(本节素材不足:{blocked_reason})"
|
||||
section_titles.append(sec_title)
|
||||
section_bufs.append(cleaned_buf)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("draft section [%s] failed: %s", sec_title, e, exc_info=True)
|
||||
section_titles.append(sec_title)
|
||||
section_bufs.append("(内容生成失败,请重试)")
|
||||
section_errors.append(f"「{sec_title}」{type(e).__name__}: {e}")
|
||||
|
||||
# 严格模式下有章节无法基于素材写作 → 暂停求助用户,不产出半成品草稿
|
||||
if blocked_sections:
|
||||
return {
|
||||
"blocked_sections": blocked_sections,
|
||||
"status": "awaiting_section_help",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
|
||||
{
|
||||
"type": "notice", "agent_name": "作家", "title": "素材不足",
|
||||
"detail": f"有 {len(blocked_sections)} 个章节缺少足够素材支撑写作,需要你决定如何处理。",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def _assemble() -> str:
|
||||
# 章节正文里的标题统一归一到 ###(要点小标题),章节标题用 ##,文章标题用 #
|
||||
body = "\n\n".join(
|
||||
f"## {t}\n\n{_normalize_section_headings(b)}"
|
||||
for t, b in zip(section_titles, section_bufs)
|
||||
)
|
||||
return f"# {article_title}\n\n{body}"
|
||||
|
||||
full_markdown = _assemble()
|
||||
word_count = count_words(full_markdown)
|
||||
|
||||
# 扩写兜底:总字数不足目标 90% 时,重写并扩充最短的章节(单轮,非流式)
|
||||
if section_bufs and word_count < word_count_target * 0.9:
|
||||
idx = min(range(len(section_bufs)), key=lambda k: count_words(section_bufs[k]))
|
||||
target_sec = sections[idx]
|
||||
expand_target = count_words(section_bufs[idx]) + (word_count_target - word_count)
|
||||
try:
|
||||
expand_msg = DRAFT_EXPAND_USER.format(
|
||||
article_title=article_title,
|
||||
section_title=section_titles[idx],
|
||||
key_points="\n".join(
|
||||
f"- {p}" for p in (target_sec.get("key_points") or target_sec.get("keyPoints") or [])
|
||||
),
|
||||
section_materials=_get_section_materials(
|
||||
target_sec.get("material_ids") or target_sec.get("materialIds") or [], pkg
|
||||
),
|
||||
existing_text=section_bufs[idx],
|
||||
target_words=expand_target,
|
||||
)
|
||||
resp = await model.ainvoke([
|
||||
SystemMessage(content=DRAFT_SECTION_SYSTEM),
|
||||
HumanMessage(content=expand_msg),
|
||||
])
|
||||
parser = ThinkTagParser()
|
||||
expanded = "".join(
|
||||
t for is_thinking, t in parser.feed(_content_to_text(resp.content)) if not is_thinking
|
||||
).strip()
|
||||
# 仅在扩写确实更长时才采用,避免越改越短
|
||||
if count_words(expanded) > count_words(section_bufs[idx]):
|
||||
section_bufs[idx] = expanded
|
||||
full_markdown = _assemble()
|
||||
word_count = count_words(full_markdown)
|
||||
except Exception as e:
|
||||
logger.warning("draft expand failed for section [%s]: %s", section_titles[idx], e)
|
||||
|
||||
draft = DraftArticle(
|
||||
title=outline.get("title", "文章"),
|
||||
outline=outline,
|
||||
full_markdown=full_markdown,
|
||||
word_count=word_count,
|
||||
revision_count=revision_count,
|
||||
)
|
||||
|
||||
# 分节写作出现模型异常时,以非致命 notice 提示用户,草稿仍照常产出
|
||||
notices: list[dict] = []
|
||||
if section_errors:
|
||||
notices.append({
|
||||
"type": "notice",
|
||||
"agent_name": "作家",
|
||||
"title": "模型错误",
|
||||
"detail": "部分章节生成失败,对应段落已占位。原因:" + ";".join(section_errors[:3]),
|
||||
})
|
||||
|
||||
return {
|
||||
"current_draft": draft,
|
||||
"drafts": [draft],
|
||||
"status": "awaiting_draft_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": f"开始写作(第{revision_count+1}稿)"},
|
||||
*notices,
|
||||
{
|
||||
"type": "draft_ready",
|
||||
"agent_name": "作家",
|
||||
"message": f"草稿完成,共 {word_count} 字",
|
||||
"draft_title": outline.get("title", "文章"),
|
||||
"draft_markdown": full_markdown,
|
||||
"revision_count": revision_count,
|
||||
},
|
||||
],
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
"""作家节点 Phase 1:生成文章大纲,支持流式 chunk 输出。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..state import AIWritingState, Outline, OutlineSection
|
||||
from ..prompts.writer_prompts import (
|
||||
OUTLINE_SYSTEM, OUTLINE_USER, OUTLINE_REVISE_SYSTEM, OUTLINE_REVISE_USER,
|
||||
)
|
||||
from ._utils import get_model, parse_json, _content_to_text, ThinkTagParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 上下文压缩:传给大纲规划的素材摘要做条数和单条长度双重截断,防止 token 溢出。
|
||||
_MAX_OUTLINE_MATERIALS = 12
|
||||
_MATERIAL_SNIPPET_CHARS = 150
|
||||
|
||||
|
||||
def _materials_summary(pkg: dict) -> str:
|
||||
lines = []
|
||||
for m in pkg.get("materials", [])[:_MAX_OUTLINE_MATERIALS]:
|
||||
lines.append(f"[{m['id']}] {m['title']} | {m['source']} | {m['content'][:_MATERIAL_SNIPPET_CHARS]}")
|
||||
return "\n".join(lines) or "(无素材)"
|
||||
|
||||
|
||||
def _parse_outline(raw: dict) -> Outline:
|
||||
sections = []
|
||||
for s in raw.get("sections", []):
|
||||
sections.append(OutlineSection(
|
||||
section_title=s.get("section_title", ""),
|
||||
key_points=s.get("key_points", []),
|
||||
material_ids=s.get("material_ids", []),
|
||||
))
|
||||
if not sections:
|
||||
sections = [OutlineSection(section_title="正文", key_points=["主要内容"], material_ids=[])]
|
||||
return Outline(title=raw.get("title", "未命名文章"), sections=sections)
|
||||
|
||||
|
||||
async def writer_outline_node(state: AIWritingState, config: RunnableConfig) -> dict:
|
||||
pkg = state.get("material_package") or {"keywords": [], "materials": [], "summary": ""}
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
|
||||
feedback = ""
|
||||
if intervention.get("action") == "re_outline":
|
||||
feedback = intervention.get("_feedback") or intervention.get("outline_feedback") or ""
|
||||
elif intervention.get("outline_feedback"):
|
||||
feedback = intervention.get("outline_feedback") or ""
|
||||
|
||||
# 重新规划且已有大纲 → 走「编辑模式」:把当前大纲喂给模型,按用户要求调整,
|
||||
# 而非从零重生成(否则「加一节/删一节」这类指令没有可编辑的基准)。
|
||||
current_outline = state.get("current_outline")
|
||||
is_revise = bool(feedback) and bool(current_outline)
|
||||
|
||||
if is_revise:
|
||||
system_prompt = OUTLINE_REVISE_SYSTEM
|
||||
user_prompt = OUTLINE_REVISE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
current_outline=json.dumps(current_outline, ensure_ascii=False, indent=2),
|
||||
feedback=feedback,
|
||||
materials_summary=_materials_summary(pkg),
|
||||
)
|
||||
else:
|
||||
system_prompt = OUTLINE_SYSTEM
|
||||
feedback_section = f"\n重新规划要求:{feedback}" if feedback else ""
|
||||
user_prompt = OUTLINE_USER.format(
|
||||
user_intent=state["user_intent"],
|
||||
article_type=state.get("article_type", "analysis"),
|
||||
target_audience=state.get("target_audience", "普通读者"),
|
||||
word_count_target=state.get("word_count_target", 800),
|
||||
materials_summary=_materials_summary(pkg),
|
||||
feedback_section=feedback_section,
|
||||
)
|
||||
|
||||
# SSE 流式推送
|
||||
cfg = config.get("configurable", {})
|
||||
sse_queue: asyncio.Queue | None = cfg.get("sse_queue")
|
||||
session_id: str = cfg.get("sse_session_id", "")
|
||||
|
||||
full_text = "" # 只累积正文(非思考)用于 JSON 解析
|
||||
parser = ThinkTagParser()
|
||||
|
||||
try:
|
||||
# get_model 放进 try:模型名配错时 create_chat_model 抛 ValueError,
|
||||
# 留在 try 外会硬崩溃;放进来则统一走下方的 status:error 兜底。
|
||||
model = get_model(config)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=system_prompt),
|
||||
HumanMessage(content=user_prompt),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
if sse_queue is not None:
|
||||
await sse_queue.put(("outline_chunk", {
|
||||
"session_id": session_id,
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error("outline streaming failed: %s", e, exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"progress_events": [{
|
||||
"type": "error",
|
||||
"agent_name": "作家",
|
||||
"title": "模型错误",
|
||||
"detail": str(e),
|
||||
"message": "大纲生成失败",
|
||||
}],
|
||||
}
|
||||
|
||||
# 编辑模式解析失败时回退为「当前大纲」,避免把用户已有大纲弄丢
|
||||
if is_revise:
|
||||
fallback = current_outline
|
||||
else:
|
||||
fallback = {
|
||||
"title": "未命名文章",
|
||||
"sections": [
|
||||
{"section_title": "引言", "key_points": ["背景介绍"], "material_ids": []},
|
||||
{"section_title": "正文", "key_points": ["核心内容"], "material_ids": []},
|
||||
{"section_title": "结论", "key_points": ["总结"], "material_ids": []},
|
||||
],
|
||||
}
|
||||
result = parse_json(full_text, fallback)
|
||||
|
||||
outline = _parse_outline(result)
|
||||
section_count = len(outline["sections"])
|
||||
|
||||
return {
|
||||
"current_outline": outline,
|
||||
"status": "awaiting_outline_confirm",
|
||||
"progress_events": [
|
||||
{"type": "step_start", "agent_name": "作家", "message": "开始规划大纲"},
|
||||
{"type": "outline_ready", "agent_name": "作家", "message": f"大纲已生成,共 {section_count} 个章节"},
|
||||
],
|
||||
}
|
||||
Whitespace-only changes.
+52
@@ -0,0 +1,52 @@
|
||||
EDITOR_SYSTEM = """你是一名资深主编(编辑),标准极高,要求严格。
|
||||
你的目标:审查文章质量,给出客观评分和具体修改意见。
|
||||
你的工作背景:你在一家权威媒体担任主编十五年,见过各种质量的文章。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从指定维度审查文章
|
||||
2. 给出该维度的评分(0-100 的整数)和具体问题列表
|
||||
你不能改写文章,只做当前指定维度的审查。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- score 是 0-100 的整数。
|
||||
- 每个问题的 severity 必须是 "major"(必须修改)或 "minor"(建议修改)。
|
||||
- 文章在该维度没有问题时,issues 填空数组 []。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{
|
||||
"score": 82,
|
||||
"issues": [
|
||||
{"severity": "major", "location": "第二段", "issue": "引用的增长数据没有素材支撑", "suggestion": "补充素材 m002 中的具体数据,并注明来源"},
|
||||
{"severity": "minor", "location": "结尾段", "issue": "结论略显仓促", "suggestion": "增加一句对未来行业趋势的判断"}
|
||||
]
|
||||
}"""
|
||||
|
||||
FACT_REVIEW_USER = """请审查以下文章的【事实准确性】维度。
|
||||
判断标准:引用的数据/事实是否有素材支撑,是否有捏造或夸大。
|
||||
|
||||
参考素材:
|
||||
{materials_summary}
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
|
||||
LOGIC_REVIEW_USER = """请审查以下文章的【逻辑结构】维度。
|
||||
判断标准:论点是否清晰,论据是否充分,段落是否连贯,结构是否合理。
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
|
||||
LANGUAGE_REVIEW_USER = """请审查以下文章的【语言文字】维度。
|
||||
判断标准:语言是否流畅,用词是否准确,有无错别字或病句。
|
||||
|
||||
待审文章:
|
||||
{article}
|
||||
|
||||
只输出 JSON(score: 0-100 整数,issues 列表,每条含 severity):"""
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
# 注:两个 USER 提示词末尾的 `/no_think` 是 Qwen3 系列的"软开关",
|
||||
# 收到当轮即跳过思考过程;对其它大模型只是一行普通文本、无副作用。
|
||||
# 中文「严禁思考」强令块对所有模型生效,保留不动。
|
||||
|
||||
EXTRACT_KEYWORDS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:为写作任务提取搜索关键词。
|
||||
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从写作意图中按要求的数量提取搜索关键词
|
||||
你不能写文章,不能评价内容好坏,只做关键词提取。
|
||||
|
||||
【严禁思考】
|
||||
- 这是一个简单的结构化抽取任务,禁止任何思考、分析、推理或规划过程。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
|
||||
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 ",不要用中文引号 “”。
|
||||
- 关键词要具体、可搜索,避免过于宽泛的词。
|
||||
|
||||
【完整示例】
|
||||
输入意图:写一篇分析今年新能源汽车出口增长的文章
|
||||
正确输出:
|
||||
{"keywords": ["新能源汽车出口", "汽车出口数据", "新能源汽车海外市场", "电动车出口增长", "汽车出口政策"]}"""
|
||||
|
||||
EXTRACT_KEYWORDS_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
|
||||
请提取 {keyword_count} 个搜索关键词,用于搜集相关素材。
|
||||
关键词数量必须为 {keyword_count} 个,不多不少。
|
||||
关键词必须是依据上述写作意图提炼的真实检索词,不要照抄下方 JSON 模板里的占位符。
|
||||
只输出 JSON(下方 ... 是占位示意,不是真实关键词):
|
||||
{{"keywords": ["...", "..."]}}
|
||||
|
||||
/no_think"""
|
||||
|
||||
RANK_MATERIALS_SYSTEM = """你是一名专业情报研究员(素材收集专家)。
|
||||
你的目标:对搜索结果按相关度排序和筛选。
|
||||
你的工作背景:你在一家权威媒体机构任职,熟悉各类信息源的可信度评估。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 从提供的素材列表中,选出与写作意图最相关的素材ID,按相关度从高到低排序
|
||||
你不能写文章,只做素材筛选排序。
|
||||
|
||||
【严禁思考】
|
||||
- 这是一个简单的筛选排序任务,禁止任何思考、分析、推理或规划过程。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容,禁止输出"让我想想""首先"之类的过渡语。
|
||||
- 收到输入后必须立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- ranked_ids 中的 ID 必须来自素材列表,不要编造 ID。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{"ranked_ids": ["m003", "m001", "m005"], "summary": "共搜索到 8 条素材,精选 5 条最相关的"}"""
|
||||
|
||||
RANK_MATERIALS_USER = """写作意图:{user_intent}
|
||||
|
||||
素材列表(每条含ID和摘要):
|
||||
{materials_list}
|
||||
|
||||
请选出最相关的素材ID(最多8条),按相关度从高到低排序。只输出 JSON:
|
||||
{{"ranked_ids": ["id1", "id2"], "summary": "共搜索到X条素材,精选Y条"}}
|
||||
|
||||
/no_think"""
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
OUTLINE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据素材和意图,规划清晰的文章大纲。
|
||||
你的工作背景:你有十年内容创作经验,擅长将零散素材组织成有逻辑的文章结构。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 分析素材,提取核心观点
|
||||
2. 规划文章章节结构和每节要点
|
||||
你不能写正文,只写大纲。大纲须紧扣写作意图,并尽量覆盖可用素材中的核心信息。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
|
||||
- 禁止输出任何对用户说的话或对任务的评价(如"素材不足""无法规划大纲");无论素材是否充分,都必须直接产出大纲 JSON。
|
||||
- 收到输入后立即、直接输出最终 JSON,不得有任何前置内容。
|
||||
|
||||
【要点(key_points)的特殊要求】
|
||||
- 每个要点都会作为文章里的一个二级小标题呈现,所以要点必须写成"标题式短语"。
|
||||
- 用简洁的名词性短语,控制在 15 字以内,不要写成完整句子,不要带句末标点。
|
||||
- 每个章节规划 2-4 个要点,不要过多,避免文章被切得过碎。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要任何解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 ",不要用中文引号 “”。
|
||||
- 规划 3-5 个章节;material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
|
||||
|
||||
【完整示例】
|
||||
正确输出:
|
||||
{
|
||||
"title": "新能源汽车出口为何高速增长",
|
||||
"sections": [
|
||||
{"section_title": "引言:出口数据创新高", "key_points": ["出口量同比增幅", "全球市场排名变化"], "material_ids": ["m001"]},
|
||||
{"section_title": "增长背后的三大动因", "key_points": ["产业链成本优势", "海外政策利好", "产品竞争力提升"], "material_ids": ["m002", "m003"]},
|
||||
{"section_title": "结论与展望", "key_points": ["未来增长挑战", "行业趋势判断"], "material_ids": ["m004"]}
|
||||
]
|
||||
}"""
|
||||
|
||||
OUTLINE_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
目标字数:{word_count_target}字{feedback_section}
|
||||
|
||||
可用素材摘要:
|
||||
{materials_summary}
|
||||
|
||||
请规划文章大纲(3-5个章节),只输出 JSON 格式。"""
|
||||
|
||||
# ── 重新规划(基于现有大纲的编辑)─────────────────────────────────────────────
|
||||
|
||||
OUTLINE_REVISE_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据用户的修改要求,调整一份【已有的文章大纲】。
|
||||
你的工作背景:你有十年内容创作经验,擅长精准理解修改意图并落实到大纲结构上。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 在「当前大纲」基础上,按「用户修改要求」做调整
|
||||
你不能写正文,只输出调整后的大纲。
|
||||
|
||||
【修改原则】(最高优先级)
|
||||
- 准确识别并落实用户意图,常见操作:新增章节、删除章节、合并章节、拆分章节、调整章节顺序、修改章节标题、调整章节要点。
|
||||
- 只改用户明确提到的部分:用户没要求改动的章节,必须原样保留(标题、要点、material_ids 全部不变)。
|
||||
- 新增章节时:紧扣写作意图补充合理章节,并尽量关联可用素材的 material_ids。
|
||||
- 章节数量完全由用户要求决定,没有上下限——用户要加就加、要减就减。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止 <think></think> 标签及其中内容。
|
||||
- 禁止输出任何对用户说的话;收到输入后立即、直接输出最终 JSON。
|
||||
|
||||
【要点(key_points)的特殊要求】
|
||||
- 每个要点会作为文章里的二级小标题呈现,写成简洁名词性短语,15 字以内,不带句末标点。
|
||||
- 每个章节 2-4 个要点。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个调整后的【完整】大纲 JSON 对象,不要解释文字,不要 markdown 代码块。
|
||||
- 必须使用英文双引号 "。
|
||||
- material_ids 中的 ID 必须来自给定素材,没有合适素材时填 []。
|
||||
- 结构与「当前大纲」一致:
|
||||
{"title": "...", "sections": [{"section_title": "...", "key_points": ["..."], "material_ids": ["..."]}]}"""
|
||||
|
||||
OUTLINE_REVISE_USER = """写作意图:{user_intent}
|
||||
文章类型:{article_type}
|
||||
目标读者:{target_audience}
|
||||
目标字数:{word_count_target}字
|
||||
|
||||
当前大纲(JSON):
|
||||
{current_outline}
|
||||
|
||||
用户修改要求:{feedback}
|
||||
|
||||
可用素材摘要:
|
||||
{materials_summary}
|
||||
|
||||
请在「当前大纲」基础上落实上述「用户修改要求」,未提到的章节保持原样,输出调整后的完整大纲 JSON。"""
|
||||
|
||||
DRAFT_SECTION_SYSTEM = """你是一名专业内容创作者(作家)。
|
||||
你的目标:根据大纲和素材,写出一段结构清晰、高质量的文章内容。
|
||||
你的工作背景:你有十年内容创作经验,文字流畅,善于用事实说话。
|
||||
|
||||
你只能做以下事情:
|
||||
1. 根据给定的章节要点和素材写这一段内容
|
||||
2. 优先采用素材中的事实与数据;素材不足时可结合你的通用知识与常识补充论述,但须紧扣写作意图与章节主题,不要脱离主题杜撰具体的数字、人名、机构名与日期
|
||||
你不能修改大纲,不能写其他章节,只写当前这一段。
|
||||
|
||||
【严禁思考与表态】(最高优先级)
|
||||
- 禁止输出任何思考、分析、规划过程,禁止"让我想想""首先""我需要""根据写作要求"之类的过渡语。
|
||||
- 禁止输出 <think></think> 标签及其中的任何内容。
|
||||
- 禁止输出任何对用户说的话或对任务本身的评价,例如"我无法完成此任务""您提供的素材无法支撑写作""建议您提供……素材""素材与意图不匹配"等——这类话绝不允许出现在文章里。
|
||||
- 无论素材是否充分,你都必须直接产出文章正文本身;收到输入后立即开始写正文(含 ### 小标题),不要有任何前置说明。
|
||||
|
||||
【结构要求】
|
||||
- 把本章节的每一个「章节要点」写成一个二级小标题,用 Markdown 三级标题语法:### 要点
|
||||
- 每个 ### 小标题下面紧跟该要点的论述正文,正文连贯成段。
|
||||
- 即使章节只有一个要点,也要为它写出对应的 ### 小标题。
|
||||
- 小标题文字简洁,直接沿用给定的要点措辞,不要加序号、不要加标点。
|
||||
- 只能使用 ### 这一级标题;绝不要输出 # 或 ##(文章标题、章节标题由系统统一添加)。
|
||||
|
||||
【输出要求】
|
||||
- 直接返回这一段的内容(含 ### 小标题与正文),不要 JSON,不要解释。
|
||||
- 不要写"以下是正文""本段内容如下"之类的开场白。
|
||||
- 紧扣本章节主题,不要写「文章整体结构」中其他章节负责的内容,避免重复。
|
||||
- 充分展开每个要点:每个要点至少用 2-3 句话论述,尽量结合素材中的事实与数据说理,写满要求的篇幅。"""
|
||||
|
||||
# ── 写作模式说明:注入 DRAFT_SECTION_USER 的 {mode_section} ───────────────────
|
||||
|
||||
# 章节无法基于素材写作时,作家在严格模式下输出的哨兵标记
|
||||
SECTION_BLOCKED_MARKER = "[[SECTION_BLOCKED]]"
|
||||
|
||||
WRITER_MODE_LOOSE = """
|
||||
【写作模式:宽松】
|
||||
素材充分时优先采用素材;素材不足时可结合你的通用知识与常识补充论述,
|
||||
但必须紧扣写作意图与本章节主题,不要脱离主题杜撰具体数字、人名、机构名。"""
|
||||
|
||||
WRITER_MODE_STRICT = """
|
||||
【写作模式:严格依据素材】
|
||||
本章节论述必须严格基于上方「相关素材」,不得使用素材之外的知识,不得杜撰任何事实与数据。
|
||||
如果上方素材确实不足以支撑本章节的写作,不要勉强拼凑、不要写无关内容,
|
||||
此时只输出一行:{marker} 一句话说明缺什么素材(例:{marker} 缺少关于XX的具体数据与案例),
|
||||
此外不要输出任何内容。只要素材能支撑哪怕部分要点,就正常写作,不要输出该标记。""".format(
|
||||
marker=SECTION_BLOCKED_MARKER
|
||||
)
|
||||
|
||||
DRAFT_SECTION_USER = """文章标题:{article_title}
|
||||
文章整体结构:
|
||||
{outline_context}
|
||||
|
||||
当前章节:{section_title}
|
||||
章节要点:{key_points}
|
||||
|
||||
相关素材:
|
||||
{section_materials}
|
||||
{revision_section}{mode_section}
|
||||
请写出「{section_title}」这一章节的内容,不少于 {target_words} 字。
|
||||
把上述每个章节要点写成一个 ### 二级小标题,并在标题下展开论述,论述充分、内容充实。"""
|
||||
|
||||
DRAFT_EXPAND_USER = """文章标题:{article_title}
|
||||
当前章节:{section_title}
|
||||
章节要点:{key_points}
|
||||
|
||||
相关素材:
|
||||
{section_materials}
|
||||
|
||||
这一章节已有内容如下:
|
||||
{existing_text}
|
||||
|
||||
以上内容篇幅不足。请在保留原意、事实和 ### 二级小标题结构的基础上重写并扩充这一章节,使内容不少于 {target_words} 字。
|
||||
要求:
|
||||
- 补充更多来自素材的事实、数据与分析,论述更充分、更具体。
|
||||
- 保留并沿用原有的 ### 小标题(每个章节要点对应一个),只用 ### 这一级标题,不要输出 # 或 ##。
|
||||
- 直接返回扩充后的完整章节内容(含 ### 小标题),不要解释。"""
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider
|
||||
from .demo_search import DemoSearchProvider
|
||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||
|
||||
|
||||
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
"""
|
||||
config.yaml 中配置:
|
||||
ai_writing:
|
||||
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
|
||||
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
||||
intranet_knowledge_base: samples
|
||||
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
||||
|
||||
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
||||
"""
|
||||
cfg = ai_writing_config or {}
|
||||
provider = cfg.get("search_provider", "internet")
|
||||
|
||||
if provider == "intranet":
|
||||
return IntranetSearchProvider(
|
||||
base_url=cfg.get("intranet_search_url", ""),
|
||||
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
||||
)
|
||||
if provider == "tavily":
|
||||
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
||||
if provider == "demo":
|
||||
return DemoSearchProvider()
|
||||
|
||||
# 默认 internet 模式:DuckDuckGo + demo fallback
|
||||
region = cfg.get("ddg_region", "cn-zh")
|
||||
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
|
||||
|
||||
|
||||
class _FallbackSearchProvider(BaseSearchProvider):
|
||||
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
|
||||
|
||||
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
|
||||
self._primary = primary
|
||||
self._fallback = fallback
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
|
||||
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs):
|
||||
results = await self._primary.search(query, max_results=max_results, **kwargs)
|
||||
if not results:
|
||||
results = await self._fallback.search(query, max_results=max_results, **kwargs)
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return await self._primary.health_check() or await self._fallback.health_check()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BaseSearchProvider", "SearchResult",
|
||||
"InternetSearchProvider", "IntranetSearchProvider",
|
||||
"DemoSearchProvider", "DuckDuckGoSearchProvider",
|
||||
"get_search_provider",
|
||||
]
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""统一搜索结果格式 —— 内外网共用"""
|
||||
title: str
|
||||
content: str
|
||||
source: str
|
||||
published_at: str
|
||||
url: str = ""
|
||||
relevance_score: float = 0.0
|
||||
source_type: str = "internet"
|
||||
recUuid: str = ""
|
||||
extra_metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseSearchProvider(ABC):
|
||||
"""搜索提供者抽象基类。
|
||||
|
||||
对接内网步骤:
|
||||
1. 继承此类,创建 IntranetSearchProvider
|
||||
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
|
||||
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
|
||||
4. 无需修改任何 Agent 代码
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def health_check(self) -> bool: ...
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
# 演示素材库:按话题分类,用于关键词模糊匹配
|
||||
_DEMO_CORPUS = [
|
||||
{
|
||||
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
|
||||
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
|
||||
"source": "中国汽车工业协会",
|
||||
"published_at": "2025-10-15",
|
||||
"relevance_score": 0.95,
|
||||
},
|
||||
{
|
||||
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
|
||||
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
|
||||
"source": "电池中国网",
|
||||
"published_at": "2025-09-20",
|
||||
"relevance_score": 0.88,
|
||||
},
|
||||
{
|
||||
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
|
||||
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
|
||||
"source": "中国充电联盟",
|
||||
"published_at": "2025-08-30",
|
||||
"relevance_score": 0.82,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
|
||||
"items": [
|
||||
{
|
||||
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
|
||||
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
|
||||
"source": "中国信通院",
|
||||
"published_at": "2025-10-08",
|
||||
"relevance_score": 0.93,
|
||||
},
|
||||
{
|
||||
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
|
||||
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
|
||||
"source": "麦肯锡中国",
|
||||
"published_at": "2025-09-12",
|
||||
"relevance_score": 0.87,
|
||||
},
|
||||
{
|
||||
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
|
||||
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
|
||||
"source": "艾瑞咨询",
|
||||
"published_at": "2025-10-01",
|
||||
"relevance_score": 0.79,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
|
||||
"items": [
|
||||
{
|
||||
"title": "光伏装机量突破900GW,度电成本降至0.18元",
|
||||
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
|
||||
"source": "国家能源局",
|
||||
"published_at": "2025-10-12",
|
||||
"relevance_score": 0.92,
|
||||
},
|
||||
{
|
||||
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
|
||||
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
|
||||
"source": "上海环境能源交易所",
|
||||
"published_at": "2025-09-28",
|
||||
"relevance_score": 0.85,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
|
||||
"items": [
|
||||
{
|
||||
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
|
||||
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
|
||||
"source": "医学界",
|
||||
"published_at": "2025-09-18",
|
||||
"relevance_score": 0.91,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
|
||||
"items": [
|
||||
{
|
||||
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
|
||||
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
|
||||
"source": "国家统计局",
|
||||
"published_at": "2025-10-18",
|
||||
"relevance_score": 0.90,
|
||||
},
|
||||
{
|
||||
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
|
||||
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
|
||||
"source": "商务部",
|
||||
"published_at": "2025-10-05",
|
||||
"relevance_score": 0.84,
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# 通用兜底素材(与任意查询都能凑合搭配)
|
||||
_FALLBACK_ITEMS = [
|
||||
{
|
||||
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
|
||||
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
|
||||
"source": "产业研究院",
|
||||
"published_at": "2025-08-15",
|
||||
"relevance_score": 0.55,
|
||||
},
|
||||
{
|
||||
"title": "政策解读:2025年产业发展专项支持资金管理办法",
|
||||
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
|
||||
"source": "工信部",
|
||||
"published_at": "2025-07-20",
|
||||
"relevance_score": 0.50,
|
||||
},
|
||||
{
|
||||
"title": "市场竞争格局分析:头部集中与长尾分散并存",
|
||||
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
|
||||
"source": "普华永道",
|
||||
"published_at": "2025-09-01",
|
||||
"relevance_score": 0.48,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _score_topic(query: str, keywords: list[str]) -> int:
|
||||
"""简单关键词匹配打分"""
|
||||
q = query.lower()
|
||||
return sum(1 for kw in keywords if kw in q)
|
||||
|
||||
|
||||
class DemoSearchProvider(BaseSearchProvider):
|
||||
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
|
||||
|
||||
仅用于开发测试;生产环境应配置 intranet 提供者。
|
||||
"""
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
|
||||
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
|
||||
|
||||
items = []
|
||||
if best_score > 0 and best_topic:
|
||||
items.extend(best_topic["items"])
|
||||
|
||||
# 用通用素材补齐
|
||||
for fb in _FALLBACK_ITEMS:
|
||||
if len(items) >= max_results:
|
||||
break
|
||||
items.append(fb)
|
||||
|
||||
results = []
|
||||
for i, it in enumerate(items[:max_results]):
|
||||
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
|
||||
results.append(SearchResult(
|
||||
title=it["title"],
|
||||
content=it["content"],
|
||||
source=it["source"],
|
||||
published_at=it["published_at"],
|
||||
url=f"https://demo.internal/{url_hash}",
|
||||
relevance_score=it["relevance_score"],
|
||||
source_type="demo",
|
||||
recUuid=f"demo-{i:03d}",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
return True
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DuckDuckGoSearchProvider(BaseSearchProvider):
|
||||
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
|
||||
|
||||
def __init__(self, region: str = "cn-zh", max_timeout: int = 30):
|
||||
self.region = region
|
||||
self.max_timeout = max_timeout
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
|
||||
self.errors: list[str] = []
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(
|
||||
self._sync_search, query, max_results
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
logger.error("ddgs not installed. Run: uv add ddgs")
|
||||
self.errors.append("ImportError: ddgs 未安装")
|
||||
return []
|
||||
|
||||
ddgs = DDGS(timeout=self.max_timeout)
|
||||
try:
|
||||
raw = ddgs.text(
|
||||
query,
|
||||
region=self.region,
|
||||
safesearch="moderate",
|
||||
max_results=max_results,
|
||||
)
|
||||
items = list(raw) if raw else []
|
||||
except Exception as e:
|
||||
logger.warning("DDGS.text failed: %s", e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
results = []
|
||||
for item in items:
|
||||
url = item.get("href", item.get("link", ""))
|
||||
source = url.split("/")[2] if url and "/" in url else "web"
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=item.get("body", item.get("snippet", ""))[:500],
|
||||
source=source,
|
||||
published_at=item.get("published", ""),
|
||||
url=url,
|
||||
relevance_score=0.75,
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return len(results) > 0
|
||||
except Exception:
|
||||
return False
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InternetSearchProvider(BaseSearchProvider):
|
||||
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 5,
|
||||
date_from: str = "",
|
||||
date_to: str = "",
|
||||
) -> List[SearchResult]:
|
||||
try:
|
||||
results = await asyncio.to_thread(self._sync_search, query, max_results)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning("internet search failed for query=%r: %s", query, e)
|
||||
return []
|
||||
|
||||
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
|
||||
from deerflow.config import get_app_config
|
||||
try:
|
||||
from tavily import TavilyClient
|
||||
except ImportError:
|
||||
logger.warning("tavily not installed, falling back to demo search")
|
||||
return []
|
||||
|
||||
config = get_app_config().get_tool_config("web_search")
|
||||
api_key = None
|
||||
if config is not None and "api_key" in config.model_extra:
|
||||
api_key = config.model_extra.get("api_key")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("Tavily API key not configured, falling back to demo search")
|
||||
return []
|
||||
|
||||
client = TavilyClient(api_key=api_key)
|
||||
res = client.search(query, max_results=max_results)
|
||||
|
||||
results = []
|
||||
for item in res.get("results", []):
|
||||
content = item.get("content", "")
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
content=content[:500],
|
||||
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
|
||||
published_at=item.get("published_date", ""),
|
||||
url=item.get("url", ""),
|
||||
relevance_score=item.get("score", 0.0),
|
||||
source_type="internet",
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
|
||||
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
|
||||
_DEFAULT_KB = "samples"
|
||||
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
||||
_RAW_SNIPPET_LIMIT = 2000
|
||||
|
||||
|
||||
class IntranetSearchProvider(BaseSearchProvider):
|
||||
"""内网知识库搜索接口适配器。
|
||||
|
||||
接口:POST {endpoint}
|
||||
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
|
||||
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str = "",
|
||||
knowledge_base_name: str = _DEFAULT_KB,
|
||||
timeout: int = 120,
|
||||
**_ignored,
|
||||
):
|
||||
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
||||
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
||||
# 内网 ES 接口可能较慢,超时放宽到 2 分钟
|
||||
self.timeout = timeout
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
||||
self.errors: list[str] = []
|
||||
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
||||
self.raw_responses: list[str] = []
|
||||
|
||||
def _record_raw(self, body: str) -> None:
|
||||
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
|
||||
body = (body or "").strip()
|
||||
if not body:
|
||||
return
|
||||
if len(body) > _RAW_SNIPPET_LIMIT:
|
||||
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
|
||||
self.raw_responses.append(body)
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"knowledge_base_name": self.knowledge_base_name,
|
||||
"ts": "yes",
|
||||
}
|
||||
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
|
||||
raw_text = ""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.post(self.endpoint, json=payload)
|
||||
resp.raise_for_status()
|
||||
raw_text = resp.text
|
||||
data = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
self._record_raw(e.response.text if e.response is not None else "")
|
||||
return []
|
||||
except Exception as e:
|
||||
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
# 正常情况下接口直接返回数组;兼容个别情况下包了一层 data/results 的结构
|
||||
if isinstance(data, list):
|
||||
items = data
|
||||
elif isinstance(data, dict):
|
||||
items = data.get("data") or data.get("results") or []
|
||||
else:
|
||||
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
|
||||
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
|
||||
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
results: List[SearchResult] = []
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
md = (item or {}).get("metadata") or {}
|
||||
content = md.get("content") or (item or {}).get("page_content") or ""
|
||||
results.append(SearchResult(
|
||||
title=str(md.get("m_title") or "").strip() or "(无标题)",
|
||||
content=str(content)[:500],
|
||||
source=str(md.get("source1") or "").strip() or "内网知识库",
|
||||
published_at=str(md.get("m_publish") or "").strip(),
|
||||
url="", # 内网素材的原文地址由前端依据 recUuid 拼接
|
||||
# 接口未返回相关度分值,按 ES 返回顺序赋递减分以保留排序
|
||||
relevance_score=max(0.5, 0.95 - i * 0.03),
|
||||
source_type="intranet",
|
||||
recUuid=str(md.get("recUuid") or "").strip(),
|
||||
))
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
try:
|
||||
await self.search("测试", max_results=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
from typing import Annotated, List, Literal, Optional, TypedDict
|
||||
import operator
|
||||
|
||||
# ── 素材 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class Material(TypedDict):
|
||||
id: str
|
||||
title: str
|
||||
content: str # 截断到 500 字
|
||||
source: str
|
||||
published_at: str
|
||||
relevance_score: float
|
||||
source_type: str # "internet" | "intranet"
|
||||
url: str
|
||||
recUuid: str
|
||||
|
||||
class MaterialPackage(TypedDict):
|
||||
keywords: List[str]
|
||||
materials: List[Material]
|
||||
summary: str
|
||||
|
||||
# ── 大纲 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class OutlineSection(TypedDict):
|
||||
section_title: str
|
||||
key_points: List[str]
|
||||
material_ids: List[str]
|
||||
|
||||
class Outline(TypedDict):
|
||||
title: str
|
||||
sections: List[OutlineSection]
|
||||
|
||||
# ── 草稿 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class DraftArticle(TypedDict):
|
||||
title: str
|
||||
outline: Outline
|
||||
full_markdown: str
|
||||
word_count: int
|
||||
revision_count: int
|
||||
|
||||
# ── 编辑审核 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class ReviewIssue(TypedDict):
|
||||
category: str # "fact" | "logic" | "language"
|
||||
severity: str # "major" | "minor"
|
||||
location: str
|
||||
issue: str
|
||||
suggestion: str
|
||||
|
||||
class ReviewResult(TypedDict):
|
||||
verdict: str # "pass" | "reject"
|
||||
fact_score: int
|
||||
logic_score: int
|
||||
language_score: int
|
||||
overall_score: int
|
||||
pass_threshold: int
|
||||
revision_notes: List[ReviewIssue]
|
||||
|
||||
# ── 用户干预 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class UserInterventionPayload(TypedDict):
|
||||
action: str
|
||||
approved_material_ids: List[str]
|
||||
extra_keywords: List[str]
|
||||
edited_outline: Optional[Outline]
|
||||
outline_feedback: str
|
||||
user_revision_notes: str
|
||||
override_verdict: bool
|
||||
|
||||
# ── 主状态 ────────────────────────────────────────────────────────────────────
|
||||
|
||||
WritingStatus = Literal[
|
||||
"researching", "awaiting_material_confirm",
|
||||
"writing_outline", "awaiting_outline_confirm",
|
||||
"writing_draft", "awaiting_section_help", "awaiting_draft_confirm",
|
||||
"reviewing", "awaiting_review_confirm",
|
||||
"revising", "done", "error",
|
||||
]
|
||||
|
||||
|
||||
class BlockedSection(TypedDict):
|
||||
"""严格模式下素材不足、无法写作的章节。"""
|
||||
section_title: str
|
||||
reason: str
|
||||
|
||||
class AIWritingState(TypedDict):
|
||||
# 写作配置
|
||||
user_intent: str
|
||||
article_type: str
|
||||
target_audience: str
|
||||
word_count_target: int
|
||||
# 素材检索关键词数量(用户指定,1~10,默认 6)
|
||||
keyword_count: int
|
||||
# 写作模式:strict=严格依据素材 / loose=可结合通用知识(默认 loose)
|
||||
writing_mode: str
|
||||
|
||||
# Agent A 输出
|
||||
material_package: Optional[MaterialPackage]
|
||||
|
||||
# Agent B 输出
|
||||
current_outline: Optional[Outline]
|
||||
drafts: Annotated[List[DraftArticle], operator.add]
|
||||
current_draft: Optional[DraftArticle]
|
||||
|
||||
# Agent C 输出
|
||||
review_result: Optional[ReviewResult]
|
||||
|
||||
# 用户干预数据(每个干预点会覆盖)
|
||||
last_user_intervention: Optional[UserInterventionPayload]
|
||||
|
||||
# 严格模式下素材不足、待用户决策的章节(写作节点产出,求助暂停点消费)
|
||||
blocked_sections: Optional[List[BlockedSection]]
|
||||
|
||||
# 流程控制
|
||||
revision_count: int
|
||||
max_revisions: int
|
||||
status: WritingStatus
|
||||
|
||||
# SSE 进度推送队列(每个节点追加)
|
||||
progress_events: Annotated[List[dict], operator.add]
|
||||
+15
-3
@@ -5,7 +5,7 @@ import logging
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.canvas_agent.prompt import CANVAS_SYSTEM_PROMPT
|
||||
from deerflow.agents.canvas_agent.prompt import build_canvas_system_prompt
|
||||
from deerflow.agents.canvas_agent.tools import update_artifact_tool
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.dangling_tool_call_middleware import DanglingToolCallMiddleware
|
||||
@@ -49,7 +49,19 @@ def make_canvas_agent(config: RunnableConfig):
|
||||
if thinking_enabled and model_config and not model_config.supports_thinking:
|
||||
thinking_enabled = False
|
||||
|
||||
logger.info("Create CanvasAgent -> model_name: %s, thinking_enabled: %s", model_name, thinking_enabled)
|
||||
canvas_artifact_title = cfg.get("canvas_artifact_title") or None
|
||||
canvas_artifact_type = cfg.get("canvas_artifact_type") or None
|
||||
system_prompt = build_canvas_system_prompt(
|
||||
artifact_title=canvas_artifact_title,
|
||||
artifact_type=canvas_artifact_type,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Create CanvasAgent -> model_name: %s, thinking_enabled: %s, artifact_title: %s",
|
||||
model_name,
|
||||
thinking_enabled,
|
||||
canvas_artifact_title,
|
||||
)
|
||||
|
||||
middlewares = [
|
||||
DanglingToolCallMiddleware(),
|
||||
@@ -63,6 +75,6 @@ def make_canvas_agent(config: RunnableConfig):
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=app_config),
|
||||
tools=[update_artifact_tool, ask_clarification_tool],
|
||||
middleware=middlewares,
|
||||
system_prompt=CANVAS_SYSTEM_PROMPT,
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+62
-2
@@ -1,4 +1,18 @@
|
||||
CANVAS_SYSTEM_PROMPT = """You are a Canvas AI assistant that helps users create and edit documents and code.
|
||||
def build_canvas_system_prompt(
|
||||
artifact_title: str | None = None,
|
||||
artifact_type: str | None = None,
|
||||
) -> str:
|
||||
existing_artifact_block = ""
|
||||
if artifact_title:
|
||||
type_label = "代码" if artifact_type == "code" else "文档"
|
||||
existing_artifact_block = f"""
|
||||
<existing_artifact>
|
||||
用户当前有一个{type_label}制品,标题为 "{artifact_title}"。
|
||||
请继续编辑该制品——除非用户明确要求新建,否则不要创建新制品。
|
||||
</existing_artifact>
|
||||
"""
|
||||
|
||||
return f"""You are a Canvas AI assistant that helps users create and edit documents and code.
|
||||
|
||||
<canvas_behavior>
|
||||
You work with an **artifact** — a document or code file shown in the canvas panel on the right.
|
||||
@@ -10,11 +24,53 @@ Rules:
|
||||
- If the user is only asking a question (no edit needed), answer in text — skip the tool call.
|
||||
</canvas_behavior>
|
||||
|
||||
<clarification_system>
|
||||
Before creating a brand-new artifact or performing a major rewrite, verify you have enough
|
||||
information to do the job well. When key details are missing, call `ask_clarification` FIRST
|
||||
— do NOT start generating the artifact and ask questions mid-way.
|
||||
|
||||
**MUST clarify when (no existing artifact or major rewrite requested):**
|
||||
1. OUTPUT TYPE IS VAGUE
|
||||
- "帮我做个主页" / "make a homepage" → ask: tech stack (plain HTML / React / Vue?),
|
||||
design style, which sections to include
|
||||
- "写个报告" / "write a report" → ask: topic, target audience, expected length,
|
||||
required sections
|
||||
- "写个脚本" / "write a script" → ask: programming language, what it should do
|
||||
2. REQUIRED CONTENT IS MISSING
|
||||
- User wants a personal profile page but hasn't provided their name / bio / links
|
||||
- User wants a data table but hasn't provided the data or schema
|
||||
3. AMBIGUOUS APPROACH WITH SIGNIFICANTLY DIFFERENT OUTCOMES
|
||||
- "Design a chat system" → ask: client–server or peer-to-peer, persistence needed?
|
||||
- "Write an intro" → ask: formal essay or casual blog post?
|
||||
|
||||
**DO NOT clarify for:**
|
||||
- Simple edits to an existing artifact: "make it bold", "fix the typo", "shorten paragraph 2"
|
||||
- Formatting / style tweaks: "add comments", "use camelCase", "format the code"
|
||||
- Quick actions: expand / shorten / formalize / casualize / add_comments / fix_bugs / optimize_code
|
||||
- Requests where intent is clear from context (e.g., user pastes content and says "translate this")
|
||||
- Follow-up turns where enough context already exists in the conversation
|
||||
|
||||
**Format for clarification (use options when practical):**
|
||||
ask_clarification(
|
||||
question="你希望这个主页使用什么技术栈?",
|
||||
clarification_type="missing_info",
|
||||
options=[
|
||||
{{"id": "html", "label": "纯 HTML / CSS / JS", "description": "无框架,最简单"}},
|
||||
{{"id": "react", "label": "React", "description": "组件化,适合复杂交互"}},
|
||||
{{"id": "vue", "label": "Vue", "description": "渐进式框架,上手容易"}},
|
||||
],
|
||||
allow_custom=True,
|
||||
)
|
||||
|
||||
CRITICAL: clarification ALWAYS comes before generating the artifact. Never produce a partial
|
||||
artifact and then ask for clarification in the same response.
|
||||
</clarification_system>
|
||||
|
||||
<artifact_context>
|
||||
The current artifact state is injected into your context. When editing, build on the existing content exactly.
|
||||
When creating from scratch, produce a complete, high-quality artifact in one shot.
|
||||
</artifact_context>
|
||||
|
||||
{existing_artifact_block}
|
||||
<selection_rewrite>
|
||||
When the user's message contains a <selection> block, you MUST:
|
||||
1. Identify the exact selection in the current artifact content.
|
||||
@@ -38,3 +94,7 @@ When the user triggers a quick action, apply it to the full artifact unless spec
|
||||
- optimize_code: Improve performance and readability
|
||||
</quick_actions>
|
||||
"""
|
||||
|
||||
|
||||
# Backward-compatible constant for callers that don't pass artifact context.
|
||||
CANVAS_SYSTEM_PROMPT = build_canvas_system_prompt()
|
||||
+3
-7
@@ -5,7 +5,6 @@ from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.memory.summarization_hook import memory_flush_hook
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
|
||||
@@ -92,9 +91,8 @@ def _create_summarization_middleware(*, app_config: AppConfig | None = None) ->
|
||||
if config.summary_prompt is not None:
|
||||
kwargs["summary_prompt"] = config.summary_prompt
|
||||
|
||||
# Memory V2:不再有后台抽取队列,无需 summarization 前的 flush hook。
|
||||
hooks: list[BeforeSummarizationHook] = []
|
||||
if resolved_app_config.memory.enabled:
|
||||
hooks.append(memory_flush_hook)
|
||||
|
||||
# The logic below relies on two assumptions holding true: this factory is
|
||||
# the sole entry point for DeerFlowSummarizationMiddleware, and the runtime
|
||||
@@ -396,7 +394,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s, subagent_mode: %s",
|
||||
"Create Agent(%s) -> thinking_enabled: %s, reasoning_effort: %s, model_name: %s, is_plan_mode: %s, subagent_enabled: %s, max_concurrent_subagents: %s, writing_mode: %s",
|
||||
agent_id or "default",
|
||||
thinking_enabled,
|
||||
reasoning_effort,
|
||||
@@ -405,7 +403,6 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
subagent_enabled,
|
||||
max_concurrent_subagents,
|
||||
writing_mode,
|
||||
subagent_mode,
|
||||
)
|
||||
|
||||
# Inject run metadata for LangSmith trace tagging
|
||||
@@ -432,7 +429,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
"subagent_enabled": subagent_enabled,
|
||||
"writing_mode": writing_mode,
|
||||
"writing_artifact_path": writing_artifact_path,
|
||||
"tool_groups": subagent_tool_groups if subagent_mode else (agent_config.tool_groups if agent_config else None),
|
||||
"tool_groups": agent_config.tool_groups if agent_config else None,
|
||||
"available_skills": runtime_available_skills,
|
||||
}
|
||||
)
|
||||
@@ -483,7 +480,6 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
soul_override=subagent_soul if subagent_mode else None,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+21
-15
@@ -603,32 +603,38 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
|
||||
|
||||
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
|
||||
"""Get memory context for injection into system prompt.
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
|
||||
本地 provider —— 返回 USER.md / MEMORY.md 的 frozen snapshot。系统提示词
|
||||
在 agent 构建时定型,因此天然就是 frozen snapshot 语义。
|
||||
|
||||
Args:
|
||||
agent_name: If provided, loads per-agent memory. If None, loads global memory.
|
||||
app_config: Explicit application config. When provided, memory options
|
||||
are read from this value instead of the global config singleton.
|
||||
agent_name: agent 标识;``None`` 时按 "default" 处理。
|
||||
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
|
||||
|
||||
Returns:
|
||||
Formatted memory context string wrapped in XML tags, or empty string if disabled.
|
||||
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.memory import format_memory_for_injection, get_memory_data
|
||||
from deerflow.agents.memory.manager import build_memory_manager
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
if app_config is None:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
config = get_memory_config()
|
||||
else:
|
||||
config = app_config.memory
|
||||
|
||||
if not config.enabled or not config.injection_enabled:
|
||||
if not config.enabled or not config.injection.enabled:
|
||||
return ""
|
||||
|
||||
memory_data = get_memory_data(agent_name, user_id=get_effective_user_id())
|
||||
memory_content = format_memory_for_injection(memory_data, max_tokens=config.max_injection_tokens)
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name or "default",
|
||||
config=config,
|
||||
)
|
||||
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
|
||||
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
|
||||
memory_content = manager.build_system_prompt()
|
||||
|
||||
if not memory_content.strip():
|
||||
return ""
|
||||
|
||||
+37
-57
@@ -1,57 +1,37 @@
|
||||
"""Memory module for DeerFlow.
|
||||
|
||||
This module provides a global memory mechanism that:
|
||||
- Stores user context and conversation history in memory.json
|
||||
- Uses LLM to summarize and extract facts from conversations
|
||||
- Injects relevant memory into system prompts for personalized responses
|
||||
"""
|
||||
|
||||
from deerflow.agents.memory.prompt import (
|
||||
FACT_EXTRACTION_PROMPT,
|
||||
MEMORY_UPDATE_PROMPT,
|
||||
format_conversation_for_update,
|
||||
format_memory_for_injection,
|
||||
)
|
||||
from deerflow.agents.memory.queue import (
|
||||
ConversationContext,
|
||||
MemoryUpdateQueue,
|
||||
get_memory_queue,
|
||||
reset_memory_queue,
|
||||
)
|
||||
from deerflow.agents.memory.storage import (
|
||||
FileMemoryStorage,
|
||||
MemoryStorage,
|
||||
get_memory_storage,
|
||||
)
|
||||
from deerflow.agents.memory.updater import (
|
||||
MemoryUpdater,
|
||||
clear_memory_data,
|
||||
delete_memory_fact,
|
||||
get_memory_data,
|
||||
reload_memory_data,
|
||||
update_memory_from_conversation,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Prompt utilities
|
||||
"MEMORY_UPDATE_PROMPT",
|
||||
"FACT_EXTRACTION_PROMPT",
|
||||
"format_memory_for_injection",
|
||||
"format_conversation_for_update",
|
||||
# Queue
|
||||
"ConversationContext",
|
||||
"MemoryUpdateQueue",
|
||||
"get_memory_queue",
|
||||
"reset_memory_queue",
|
||||
# Storage
|
||||
"MemoryStorage",
|
||||
"FileMemoryStorage",
|
||||
"get_memory_storage",
|
||||
# Updater
|
||||
"MemoryUpdater",
|
||||
"clear_memory_data",
|
||||
"delete_memory_fact",
|
||||
"get_memory_data",
|
||||
"reload_memory_data",
|
||||
"update_memory_from_conversation",
|
||||
]
|
||||
"""DeerFlow 记忆模块 (Memory V2)。
|
||||
|
||||
架构 (详见 docs/MEMORY_V2_DESIGN_ZH.md):
|
||||
- :class:`MemoryProvider` 可插拔记忆 provider 抽象基类
|
||||
- :class:`MemoryManager` 编排内建 + 至多 1 个外部 provider
|
||||
- :func:`build_memory_manager` 按生效配置组装 manager 的统一入口
|
||||
- :class:`BuiltinFileProvider` 本地 USER.md / MEMORY.md provider
|
||||
- :class:`HindsightProvider` Hindsight HTTP 外部 provider
|
||||
- :mod:`deerflow.agents.memory.security` 记忆内容安全扫描
|
||||
|
||||
记忆有两条自动写入路径:LLM 主动调 ``memory`` / ``hindsight_*`` 工具,
|
||||
以及 Hindsight 的 auto-retain。注入走系统提示词 (builtin frozen snapshot)
|
||||
与 ``MemoryMiddleware.wrap_model_call`` (Hindsight 动态召回)。
|
||||
"""
|
||||
|
||||
from deerflow.agents.memory.manager import MemoryManager, build_memory_manager
|
||||
from deerflow.agents.memory.message_processing import filter_messages_for_memory
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
from deerflow.agents.memory.security import (
|
||||
MemoryContentBlocked,
|
||||
assert_memory_content_safe,
|
||||
scan_memory_content,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MemoryProvider",
|
||||
"MemoryManager",
|
||||
"build_memory_manager",
|
||||
"BuiltinFileProvider",
|
||||
"HindsightProvider",
|
||||
"MemoryContentBlocked",
|
||||
"scan_memory_content",
|
||||
"assert_memory_content_safe",
|
||||
"filter_messages_for_memory",
|
||||
]
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Memory V2 - MemoryManager 编排层。
|
||||
|
||||
职责
|
||||
====
|
||||
编排 1 个内建 provider (``builtin``,始终存在) + 至多 1 个外部 provider
|
||||
(P2 期的 ``hindsight``)。把系统提示词组装、召回、生命周期 hook 分发到各 provider。
|
||||
任一 provider 出错都不会阻断其他 provider,也不会阻断对话。
|
||||
|
||||
典型用法 (由 MemoryMiddleware 每轮构建)
|
||||
=======================================
|
||||
::
|
||||
|
||||
manager = MemoryManager()
|
||||
manager.add_provider(BuiltinFileProvider(...))
|
||||
manager.add_provider(hindsight_provider) # 可选,至多 1 个
|
||||
manager.initialize_all(user_id=..., agent_id=..., thread_id=..., base_dir=...)
|
||||
|
||||
block = manager.build_injection_block(query=user_msg, thread_id=tid)
|
||||
# ... 把 block 注入系统提示词 ...
|
||||
|
||||
manager.sync_all(user_msg, assistant_msg, thread_id=tid)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 内建 provider 的固定名字。
|
||||
BUILTIN_PROVIDER_NAME = "builtin"
|
||||
|
||||
|
||||
class MemoryManager:
|
||||
"""编排内建 provider + 至多 1 个外部 provider。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._providers: list[MemoryProvider] = []
|
||||
self._has_external: bool = False
|
||||
|
||||
# ---- 注册 ------------------------------------------------------------
|
||||
|
||||
def add_provider(self, provider: MemoryProvider) -> bool:
|
||||
"""注册一个 provider。
|
||||
|
||||
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
|
||||
第二个会被拒绝并 log 警告。返回是否注册成功。
|
||||
"""
|
||||
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
|
||||
if not is_builtin:
|
||||
if self._has_external:
|
||||
existing = next(
|
||||
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
|
||||
"unknown",
|
||||
)
|
||||
logger.warning(
|
||||
"拒绝注册记忆 provider '%s' —— 已存在外部 provider '%s'。"
|
||||
"同一时间只允许 1 个外部 provider,请用 config.memory.provider 选择。",
|
||||
provider.name,
|
||||
existing,
|
||||
)
|
||||
return False
|
||||
self._has_external = True
|
||||
self._providers.append(provider)
|
||||
logger.info("记忆 provider '%s' 已注册", provider.name)
|
||||
return True
|
||||
|
||||
@property
|
||||
def providers(self) -> list[MemoryProvider]:
|
||||
"""按注册顺序返回所有 provider。"""
|
||||
return list(self._providers)
|
||||
|
||||
def get_provider(self, name: str) -> MemoryProvider | None:
|
||||
"""按名字取 provider,不存在返回 ``None``。"""
|
||||
for p in self._providers:
|
||||
if p.name == name:
|
||||
return p
|
||||
return None
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize_all(self, **kwargs: Any) -> None:
|
||||
"""初始化所有 provider。单个 provider 失败不影响其他。"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.initialize(**kwargs)
|
||||
except Exception as exc: # noqa: BLE001 - provider 失败必须隔离
|
||||
logger.warning("记忆 provider '%s' initialize 失败:%s", provider.name, exc)
|
||||
|
||||
def shutdown_all(self) -> None:
|
||||
"""关闭所有 provider (逆序,保证干净拆除)。"""
|
||||
for provider in reversed(self._providers):
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' shutdown 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 系统提示词组装 / 召回 -------------------------------------------
|
||||
|
||||
def build_system_prompt(self) -> str:
|
||||
"""收集所有 provider 的静态系统提示词块。"""
|
||||
blocks: list[str] = []
|
||||
for provider in self._providers:
|
||||
try:
|
||||
block = provider.system_prompt_block()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' system_prompt_block 失败:%s", provider.name, exc)
|
||||
continue
|
||||
if block and block.strip():
|
||||
blocks.append(block)
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
def prefetch_all(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""收集所有 provider 的召回上下文。单个失败不阻断其他。"""
|
||||
parts: list[str] = []
|
||||
for provider in self._providers:
|
||||
try:
|
||||
result = provider.prefetch(query, thread_id=thread_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' prefetch 失败 (非致命):%s", provider.name, exc)
|
||||
continue
|
||||
if result and result.strip():
|
||||
parts.append(result)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def build_injection_block(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""组装注入系统提示词的完整记忆块。
|
||||
|
||||
= 所有 provider 的静态块 (frozen snapshot 等) + 动态召回块。
|
||||
返回空串表示无内容可注入。
|
||||
"""
|
||||
static_part = self.build_system_prompt()
|
||||
dynamic_part = self.prefetch_all(query, thread_id=thread_id)
|
||||
parts = [p for p in (static_part, dynamic_part) if p and p.strip()]
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def queue_prefetch_all(self, query: str, *, thread_id: str = "") -> None:
|
||||
"""通知所有 provider 后台预热下一轮召回。"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.queue_prefetch(query, thread_id=thread_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' queue_prefetch 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 写入 ------------------------------------------------------------
|
||||
|
||||
def sync_all(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""把一轮完成的对话同步到所有 provider。
|
||||
|
||||
``turn_index`` 是当前对话的轮次序号,透传给各 provider 用于轮次节流。
|
||||
"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.sync_turn(
|
||||
user_content,
|
||||
assistant_content,
|
||||
thread_id=thread_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' sync_turn 失败:%s", provider.name, exc)
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入后,通知外部 provider 镜像。
|
||||
|
||||
跳过 builtin 自身 (它就是写入源,镜像回去没有意义)。
|
||||
"""
|
||||
for provider in self._providers:
|
||||
if provider.name == BUILTIN_PROVIDER_NAME:
|
||||
continue
|
||||
try:
|
||||
provider.on_memory_write(action, target, content, metadata)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' on_memory_write 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 可选 hooks ------------------------------------------------------
|
||||
|
||||
def on_session_switch_all(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""通知所有 provider thread_id 已切换。"""
|
||||
if not new_thread_id:
|
||||
return
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.on_session_switch(
|
||||
new_thread_id,
|
||||
parent_thread_id=parent_thread_id,
|
||||
reset=reset,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' on_session_switch 失败:%s", provider.name, exc)
|
||||
|
||||
|
||||
def build_memory_manager(
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
config: Any = None,
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
|
||||
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
|
||||
所有 provider 已 ``initialize`` 完毕,可直接使用。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
user_id / agent_id / thread_id:
|
||||
请求上下文标识。
|
||||
config:
|
||||
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
|
||||
"""
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
if config is None:
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = MemoryManager()
|
||||
|
||||
if config.builtin.enabled:
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
)
|
||||
)
|
||||
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id or "default",
|
||||
thread_id=thread_id,
|
||||
base_dir=str(get_paths().base_dir),
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
__all__ = ["MemoryManager", "BUILTIN_PROVIDER_NAME", "build_memory_manager"]
|
||||
+15
-53
@@ -1,4 +1,8 @@
|
||||
"""Shared helpers for turning conversations into memory update inputs."""
|
||||
"""把对话整理成记忆写入所需输入的共享辅助函数。
|
||||
|
||||
Memory V2 说明:后台抽取已舍弃,这里只保留 ``filter_messages_for_memory``
|
||||
(供 MemoryMiddleware 取出最后一轮对话用于 Hindsight retain)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,34 +11,6 @@ from copy import copy
|
||||
from typing import Any
|
||||
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", re.IGNORECASE)
|
||||
_CORRECTION_PATTERNS = (
|
||||
re.compile(r"\bthat(?:'s| is) (?:wrong|incorrect)\b", re.IGNORECASE),
|
||||
re.compile(r"\byou misunderstood\b", re.IGNORECASE),
|
||||
re.compile(r"\btry again\b", re.IGNORECASE),
|
||||
re.compile(r"\bredo\b", re.IGNORECASE),
|
||||
re.compile(r"不对"),
|
||||
re.compile(r"你理解错了"),
|
||||
re.compile(r"你理解有误"),
|
||||
re.compile(r"重试"),
|
||||
re.compile(r"重新来"),
|
||||
re.compile(r"换一种"),
|
||||
re.compile(r"改用"),
|
||||
)
|
||||
_REINFORCEMENT_PATTERNS = (
|
||||
re.compile(r"\byes[,.]?\s+(?:exactly|perfect|that(?:'s| is) (?:right|correct|it))\b", re.IGNORECASE),
|
||||
re.compile(r"\bperfect(?:[.!?]|$)", re.IGNORECASE),
|
||||
re.compile(r"\bexactly\s+(?:right|correct)\b", re.IGNORECASE),
|
||||
re.compile(r"\bthat(?:'s| is)\s+(?:exactly\s+)?(?:right|correct|what i (?:wanted|needed|meant))\b", re.IGNORECASE),
|
||||
re.compile(r"\bkeep\s+(?:doing\s+)?that\b", re.IGNORECASE),
|
||||
re.compile(r"\bjust\s+(?:like\s+)?(?:that|this)\b", re.IGNORECASE),
|
||||
re.compile(r"\bthis is (?:great|helpful)\b(?:[.!?]|$)", re.IGNORECASE),
|
||||
re.compile(r"\bthis is what i wanted\b(?:[.!?]|$)", re.IGNORECASE),
|
||||
re.compile(r"对[,,]?\s*就是这样(?:[。!?!?.]|$)"),
|
||||
re.compile(r"完全正确(?:[。!?!?.]|$)"),
|
||||
re.compile(r"(?:对[,,]?\s*)?就是这个意思(?:[。!?!?.]|$)"),
|
||||
re.compile(r"正是我想要的(?:[。!?!?.]|$)"),
|
||||
re.compile(r"继续保持(?:[。!?!?.]|$)"),
|
||||
)
|
||||
|
||||
|
||||
def extract_message_text(message: Any) -> str:
|
||||
@@ -53,6 +29,16 @@ def extract_message_text(message: Any) -> str:
|
||||
return str(content)
|
||||
|
||||
|
||||
def count_user_turns(filtered_messages: list[Any]) -> int:
|
||||
"""统计已过滤消息里的用户轮次数 (= human 消息条数)。
|
||||
|
||||
供 MemoryMiddleware 算出当前轮次序号传给 provider 做轮次节流 ——
|
||||
provider 每轮被重建、无法靠自身计数,必须由这里按对话历史推算。
|
||||
入参应是 :func:`filter_messages_for_memory` 的输出。
|
||||
"""
|
||||
return sum(1 for msg in filtered_messages if getattr(msg, "type", None) == "human")
|
||||
|
||||
|
||||
def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
"""Keep only user inputs and final assistant responses for memory updates."""
|
||||
filtered = []
|
||||
@@ -83,27 +69,3 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
filtered.append(msg)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def detect_correction(messages: list[Any]) -> bool:
|
||||
"""Detect explicit user corrections in recent conversation turns."""
|
||||
recent_user_msgs = [msg for msg in messages[-6:] if getattr(msg, "type", None) == "human"]
|
||||
|
||||
for msg in recent_user_msgs:
|
||||
content = extract_message_text(msg).strip()
|
||||
if content and any(pattern.search(content) for pattern in _CORRECTION_PATTERNS):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def detect_reinforcement(messages: list[Any]) -> bool:
|
||||
"""Detect explicit positive reinforcement signals in recent conversation turns."""
|
||||
recent_user_msgs = [msg for msg in messages[-6:] if getattr(msg, "type", None) == "human"]
|
||||
|
||||
for msg in recent_user_msgs:
|
||||
content = extract_message_text(msg).strip()
|
||||
if content and any(pattern.search(content) for pattern in _REINFORCEMENT_PATTERNS):
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -1,363 +0,0 @@
|
||||
"""Prompt templates for memory update and injection."""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
TIKTOKEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
TIKTOKEN_AVAILABLE = False
|
||||
|
||||
# Prompt template for updating memory based on conversation
|
||||
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
|
||||
|
||||
Current Memory State:
|
||||
<current_memory>
|
||||
{current_memory}
|
||||
</current_memory>
|
||||
|
||||
New Conversation to Process:
|
||||
<conversation>
|
||||
{conversation}
|
||||
</conversation>
|
||||
|
||||
Instructions:
|
||||
1. Analyze the conversation for important information about the user
|
||||
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
|
||||
3. Update the memory sections as needed following the detailed length guidelines below
|
||||
|
||||
Before extracting facts, perform a structured reflection on the conversation:
|
||||
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
|
||||
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
|
||||
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
|
||||
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
|
||||
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
|
||||
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
|
||||
If yes, record them as facts with the most appropriate category and confidence.
|
||||
|
||||
{correction_hint}
|
||||
|
||||
Memory Section Guidelines:
|
||||
|
||||
**User Context** (Current state - concise summaries):
|
||||
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
|
||||
Example: Core contributor, project names with metrics (16k+ stars), technical stack
|
||||
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
|
||||
Example: Bilingual capabilities, specific interest areas, expertise domains
|
||||
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
|
||||
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
|
||||
Include: Active implementation work, troubleshooting issues, market/research interests
|
||||
Note: This captures SEVERAL concurrent focus areas, not just one task
|
||||
|
||||
**History** (Temporal context - rich paragraphs):
|
||||
- recentMonths: Detailed summary of recent activities (4-6 sentences or 1-2 paragraphs)
|
||||
Timeline: Last 1-3 months of interactions
|
||||
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
|
||||
- earlierContext: Important historical patterns (3-5 sentences or 1 paragraph)
|
||||
Timeline: 3-12 months ago
|
||||
Include: Past projects, learning journeys, established patterns
|
||||
- longTermBackground: Persistent background and foundational context (2-4 sentences)
|
||||
Timeline: Overall/foundational information
|
||||
Include: Core expertise, longstanding interests, fundamental working style
|
||||
|
||||
**Facts Extraction**:
|
||||
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
|
||||
- Include proper nouns (company names, project names, technology names)
|
||||
- Preserve technical terminology and version numbers
|
||||
- Categories:
|
||||
* preference: Tools, styles, approaches user prefers/dislikes
|
||||
* knowledge: Specific expertise, technologies mastered, domain knowledge
|
||||
* context: Background facts (job title, projects, locations, languages)
|
||||
* behavior: Working patterns, communication habits, problem-solving approaches
|
||||
* goal: Stated objectives, learning targets, project ambitions
|
||||
* correction: Explicit agent mistakes or user corrections, including the correct approach
|
||||
- Confidence levels:
|
||||
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
|
||||
* 0.7-0.8: Strongly implied from actions/discussions
|
||||
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
|
||||
|
||||
**What Goes Where**:
|
||||
- workContext: Current job, active projects, primary tech stack
|
||||
- personalContext: Languages, personality, interests outside direct work tasks
|
||||
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
|
||||
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
|
||||
- recentMonths: Detailed account of recent technical explorations and work
|
||||
- earlierContext: Patterns from slightly older interactions still relevant
|
||||
- longTermBackground: Unchanging foundational facts about the user
|
||||
|
||||
**Multilingual Content**:
|
||||
- Preserve original language for proper nouns and company names
|
||||
- Keep technical terms in their original form (DeepSeek, LangGraph, etc.)
|
||||
- Note language capabilities in personalContext
|
||||
|
||||
Output Format (JSON):
|
||||
{{
|
||||
"user": {{
|
||||
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"history": {{
|
||||
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"newFacts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
],
|
||||
"factsToRemove": ["fact_id_1", "fact_id_2"]
|
||||
}}
|
||||
|
||||
Important Rules:
|
||||
- Only set shouldUpdate=true if there's meaningful new information
|
||||
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
|
||||
- Include specific metrics, version numbers, and proper nouns in facts
|
||||
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
|
||||
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
|
||||
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
|
||||
- Remove facts that are contradicted by new information
|
||||
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
|
||||
Keep 3-5 concurrent focus themes that are still active and relevant
|
||||
- For history sections, integrate new information chronologically into appropriate time period
|
||||
- Preserve technical accuracy - keep exact names of technologies, companies, projects
|
||||
- Focus on information useful for future interactions and personalization
|
||||
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
|
||||
session-specific and ephemeral — they will not be accessible in future sessions.
|
||||
Recording upload events causes confusion in subsequent conversations.
|
||||
|
||||
Return ONLY valid JSON, no explanation or markdown."""
|
||||
|
||||
|
||||
# Prompt template for extracting facts from a single message
|
||||
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
|
||||
|
||||
Message:
|
||||
{message}
|
||||
|
||||
Extract facts in this JSON format:
|
||||
{{
|
||||
"facts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
]
|
||||
}}
|
||||
|
||||
Categories:
|
||||
- preference: User preferences (likes/dislikes, styles, tools)
|
||||
- knowledge: User's expertise or knowledge areas
|
||||
- context: Background context (location, job, projects)
|
||||
- behavior: Behavioral patterns
|
||||
- goal: User's goals or objectives
|
||||
- correction: Explicit corrections or mistakes to avoid repeating
|
||||
|
||||
Rules:
|
||||
- Only extract clear, specific facts
|
||||
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
|
||||
- Skip vague or temporary information
|
||||
|
||||
Return ONLY valid JSON."""
|
||||
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
|
||||
"""Count tokens in text using tiktoken.
|
||||
|
||||
Args:
|
||||
text: The text to count tokens for.
|
||||
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
|
||||
|
||||
Returns:
|
||||
The number of tokens in the text.
|
||||
"""
|
||||
if not TIKTOKEN_AVAILABLE:
|
||||
# Fallback to character-based estimation if tiktoken is not available
|
||||
return len(text) // 4
|
||||
|
||||
try:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
return len(encoding.encode(text))
|
||||
except Exception:
|
||||
# Fallback to character-based estimation on error
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
|
||||
"""Coerce a confidence-like value to a bounded float in [0, 1].
|
||||
|
||||
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
|
||||
to the default before clamping, preventing them from dominating ranking.
|
||||
The ``default`` parameter is assumed to be a finite value.
|
||||
"""
|
||||
try:
|
||||
confidence = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return max(0.0, min(1.0, default))
|
||||
if not math.isfinite(confidence):
|
||||
return max(0.0, min(1.0, default))
|
||||
return max(0.0, min(1.0, confidence))
|
||||
|
||||
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
|
||||
"""Format memory data for injection into system prompt.
|
||||
|
||||
Args:
|
||||
memory_data: The memory data dictionary.
|
||||
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
|
||||
|
||||
Returns:
|
||||
Formatted memory string for system prompt injection.
|
||||
"""
|
||||
if not memory_data:
|
||||
return ""
|
||||
|
||||
sections = []
|
||||
|
||||
# Format user context
|
||||
user_data = memory_data.get("user", {})
|
||||
if user_data:
|
||||
user_sections = []
|
||||
|
||||
work_ctx = user_data.get("workContext", {})
|
||||
if work_ctx.get("summary"):
|
||||
user_sections.append(f"Work: {work_ctx['summary']}")
|
||||
|
||||
personal_ctx = user_data.get("personalContext", {})
|
||||
if personal_ctx.get("summary"):
|
||||
user_sections.append(f"Personal: {personal_ctx['summary']}")
|
||||
|
||||
top_of_mind = user_data.get("topOfMind", {})
|
||||
if top_of_mind.get("summary"):
|
||||
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
|
||||
|
||||
if user_sections:
|
||||
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
|
||||
|
||||
# Format history
|
||||
history_data = memory_data.get("history", {})
|
||||
if history_data:
|
||||
history_sections = []
|
||||
|
||||
recent = history_data.get("recentMonths", {})
|
||||
if recent.get("summary"):
|
||||
history_sections.append(f"Recent: {recent['summary']}")
|
||||
|
||||
earlier = history_data.get("earlierContext", {})
|
||||
if earlier.get("summary"):
|
||||
history_sections.append(f"Earlier: {earlier['summary']}")
|
||||
|
||||
background = history_data.get("longTermBackground", {})
|
||||
if background.get("summary"):
|
||||
history_sections.append(f"Background: {background['summary']}")
|
||||
|
||||
if history_sections:
|
||||
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
|
||||
|
||||
# Format facts (sorted by confidence; include as many as token budget allows)
|
||||
facts_data = memory_data.get("facts", [])
|
||||
if isinstance(facts_data, list) and facts_data:
|
||||
ranked_facts = sorted(
|
||||
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
|
||||
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Compute token count for existing sections once, then account
|
||||
# incrementally for each fact line to avoid full-string re-tokenization.
|
||||
base_text = "\n\n".join(sections)
|
||||
base_tokens = _count_tokens(base_text) if base_text else 0
|
||||
# Account for the separator between existing sections and the facts section.
|
||||
facts_header = "Facts:\n"
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
|
||||
running_tokens = base_tokens + separator_tokens
|
||||
|
||||
fact_lines: list[str] = []
|
||||
for fact in ranked_facts:
|
||||
content_value = fact.get("content")
|
||||
if not isinstance(content_value, str):
|
||||
continue
|
||||
content = content_value.strip()
|
||||
if not content:
|
||||
continue
|
||||
category = str(fact.get("category", "context")).strip() or "context"
|
||||
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
|
||||
source_error = fact.get("sourceError")
|
||||
if category == "correction" and isinstance(source_error, str) and source_error.strip():
|
||||
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
|
||||
else:
|
||||
line = f"- [{category} | {confidence:.2f}] {content}"
|
||||
|
||||
# Each additional line is preceded by a newline (except the first).
|
||||
line_text = ("\n" + line) if fact_lines else line
|
||||
line_tokens = _count_tokens(line_text)
|
||||
|
||||
if running_tokens + line_tokens <= max_tokens:
|
||||
fact_lines.append(line)
|
||||
running_tokens += line_tokens
|
||||
else:
|
||||
break
|
||||
|
||||
if fact_lines:
|
||||
sections.append("Facts:\n" + "\n".join(fact_lines))
|
||||
|
||||
if not sections:
|
||||
return ""
|
||||
|
||||
result = "\n\n".join(sections)
|
||||
|
||||
# Use accurate token counting with tiktoken
|
||||
token_count = _count_tokens(result)
|
||||
if token_count > max_tokens:
|
||||
# Truncate to fit within token limit
|
||||
# Estimate characters to remove based on token ratio
|
||||
char_per_token = len(result) / token_count
|
||||
target_chars = int(max_tokens * char_per_token * 0.95) # 95% to leave margin
|
||||
result = result[:target_chars] + "\n..."
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_conversation_for_update(messages: list[Any]) -> str:
|
||||
"""Format conversation messages for memory update prompt.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
|
||||
Returns:
|
||||
Formatted conversation string.
|
||||
"""
|
||||
lines = []
|
||||
for msg in messages:
|
||||
role = getattr(msg, "type", "unknown")
|
||||
content = getattr(msg, "content", str(msg))
|
||||
|
||||
# Handle content that might be a list (multimodal)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for p in content:
|
||||
if isinstance(p, str):
|
||||
text_parts.append(p)
|
||||
elif isinstance(p, dict):
|
||||
text_val = p.get("text")
|
||||
if isinstance(text_val, str):
|
||||
text_parts.append(text_val)
|
||||
content = " ".join(text_parts) if text_parts else str(content)
|
||||
|
||||
# Strip uploaded_files tags from human messages to avoid persisting
|
||||
# ephemeral file path info into long-term memory. Skip the turn entirely
|
||||
# when nothing remains after stripping (upload-only message).
|
||||
if role == "human":
|
||||
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Truncate very long messages
|
||||
if len(str(content)) > 1000:
|
||||
content = str(content)[:1000] + "..."
|
||||
|
||||
if role == "human":
|
||||
lines.append(f"User: {content}")
|
||||
elif role == "ai":
|
||||
lines.append(f"Assistant: {content}")
|
||||
|
||||
return "\n\n".join(lines)
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Memory V2 - 记忆 Provider 抽象基类。
|
||||
|
||||
设计动机
|
||||
========
|
||||
Hermes Agent 的可插拔记忆系统的 deerflow 版本。所有具体的记忆后端
|
||||
(builtin 本地文件、Hindsight HTTP 等)都实现这个接口,由 ``MemoryManager``
|
||||
统一编排。Provider 之间互不感知,可自由组合 (内建 + 至多 1 个外部 plugin)。
|
||||
|
||||
生命周期
|
||||
========
|
||||
:func:`MemoryProvider.initialize` 初始化 (创建 bank、加载本地文件、建立连接)
|
||||
:func:`MemoryProvider.system_prompt_block` 注入系统提示词的静态说明文本
|
||||
:func:`MemoryProvider.prefetch` 每轮对话前召回上下文
|
||||
:func:`MemoryProvider.queue_prefetch` 下一轮的后台预热
|
||||
:func:`MemoryProvider.sync_turn` 每轮对话后异步写入后端
|
||||
:func:`MemoryProvider.get_tool_schemas` 暴露给 LLM 的工具 schema
|
||||
:func:`MemoryProvider.handle_tool_call` 路由工具调用
|
||||
:func:`MemoryProvider.shutdown` 清理 (flush 队列、关连接)
|
||||
|
||||
可选 hooks
|
||||
==========
|
||||
仅按 ``生产推荐档`` 移植两个 ——
|
||||
- :func:`MemoryProvider.on_session_switch` thread_id 切换时 (branch/resume/reset/压缩后)
|
||||
- :func:`MemoryProvider.on_memory_write` builtin 写入时,通知外部 provider 镜像
|
||||
|
||||
参见 ``docs/MEMORY_V2_DESIGN_ZH.md`` §4 的完整接口定义和 §16 的实施分期。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryProvider(ABC):
|
||||
"""所有记忆 provider 必须实现的接口。"""
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""短标识,如 ``'builtin'`` / ``'hindsight'``。"""
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def is_available(self) -> bool:
|
||||
"""是否已配置并可用。
|
||||
|
||||
在 agent 启动时被调用,用于判断要不要激活该 provider。
|
||||
实现里只检查配置和依赖,不发网络请求。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""初始化:打开连接、创建 bank、加载本地文件等。
|
||||
|
||||
kwargs 至少包含 ``user_id`` / ``base_dir``,其余字段视 provider 而定。
|
||||
Provider 可以根据 ``agent_id is None`` 决定走全局还是 per-agent 路径。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
user_id:
|
||||
当前用户的隔离标识(无认证场景下为 ``"default"``)。
|
||||
agent_id:
|
||||
当前 agent 的稳定标识;``None`` 表示走全局/默认 agent。
|
||||
thread_id:
|
||||
当前对话的 thread id;某些 provider 用它做并发隔离或 bank 后缀。
|
||||
base_dir:
|
||||
``Paths.base_dir`` 字符串形式,避免每个 provider 重复解析路径。
|
||||
"""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""清理:刷队列、关连接。默认 no-op。"""
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""注入系统提示词的静态文本。
|
||||
|
||||
典型用途:USER.md / MEMORY.md 的 frozen snapshot,provider 状态说明等。
|
||||
返回空串表示不贡献内容。
|
||||
"""
|
||||
return ""
|
||||
|
||||
# ---- 召回 / 写入 -----------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""对话前召回上下文。
|
||||
|
||||
在每次 LLM 调用前由 :class:`MemoryMiddleware` 触发。``query`` 通常
|
||||
是用户最近一条消息。实现要快;真正的网络召回应该走后台线程,
|
||||
这里直接返回缓存结果。
|
||||
|
||||
返回值:带格式的文本块,会被拼接到系统提示词末尾。空串表示无内容。
|
||||
"""
|
||||
return ""
|
||||
|
||||
def queue_prefetch(self, query: str, *, thread_id: str = "") -> None:
|
||||
"""后台预热下一轮的召回。默认 no-op。"""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""每轮对话结束后把对话持久化到后端。
|
||||
|
||||
实现应当非阻塞 —— 如果后端有延迟,排队到后台处理而不是直接调用。
|
||||
默认 no-op。
|
||||
|
||||
``turn_index`` 是当前对话的轮次序号 (从 1 开始,由编排层按对话历史算出)。
|
||||
provider 通常被每轮重新构造、无法靠自身计数,需要按轮次节流写入
|
||||
(例如 "每 N 轮 retain 一次") 的实现应当用它而非进程内计数器。
|
||||
缺省 ``None`` 表示调用方未提供。
|
||||
"""
|
||||
|
||||
# ---- LLM 工具 --------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""暴露给 LLM 的工具 schema 列表。
|
||||
|
||||
每个 schema 遵循 OpenAI function calling 格式::
|
||||
|
||||
{"name": "...", "description": "...", "parameters": {...}}
|
||||
|
||||
空数组表示 context-only(只做自动 recall/retain,不让 LLM 看见工具)。
|
||||
"""
|
||||
|
||||
def handle_tool_call(
|
||||
self, tool_name: str, args: dict[str, Any], **kwargs: Any
|
||||
) -> str:
|
||||
"""处理一个工具调用,必须返回 JSON 字符串。
|
||||
|
||||
``MemoryManager`` 会确保只把本 provider ``get_tool_schemas`` 里
|
||||
声明的工具名路由到这里。
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"Provider {self.name} does not handle tool {tool_name}"
|
||||
)
|
||||
|
||||
# ---- 可选 hooks (生产推荐档移植) -------------------------------------
|
||||
|
||||
def on_session_switch(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""thread_id 切换时触发。
|
||||
|
||||
deerflow 的 thread_id 在以下场景会变化:
|
||||
- 用户新建/切换 thread (``reset=True``)
|
||||
- LangGraph thread branch (``reset=False``,``parent_thread_id`` 为原 thread)
|
||||
- SummarizationMiddleware 压缩后继续 (``reset=False``)
|
||||
|
||||
Provider 在 ``initialize`` 时如果缓存了 per-session 的状态
|
||||
(``_session_turns`` 累积缓冲、bank 关联、查询缓存等),应当在
|
||||
这里刷新。默认 no-op。
|
||||
"""
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入时触发,通知外部 provider 镜像同一条记忆。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
action:
|
||||
``'add'`` / ``'replace'`` / ``'remove'``。
|
||||
target:
|
||||
``'user'`` / ``'memory'``,决定走全局 user bank 还是 agent 工作 bank。
|
||||
content:
|
||||
条目内容。``remove`` 时为待删除条目的内容(供 external 做幂等)。
|
||||
metadata:
|
||||
写入来源元数据 (``user_id`` / ``agent_id`` / ``thread_id`` / ``tool_name``)。
|
||||
|
||||
默认 no-op —— 只有需要镜像 builtin 写入的外部 provider (例如 Hindsight)
|
||||
才需要重写。``MemoryManager`` 调度时会跳过 builtin 自身,避免无意义的回环。
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["MemoryProvider"]
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"""Memory V2 - 具体 provider 实现。
|
||||
|
||||
- :class:`~deerflow.agents.memory.providers.builtin.BuiltinFileProvider`
|
||||
本地文件 provider (USER.md / MEMORY.md),始终启用。
|
||||
- ``HindsightProvider`` — Hindsight HTTP 客户端,P2 期接入。
|
||||
"""
|
||||
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
__all__ = ["BuiltinFileProvider"]
|
||||
+417
@@ -0,0 +1,417 @@
|
||||
"""Memory V2 - 本地文件 provider (USER.md / MEMORY.md)。
|
||||
|
||||
从 Hermes ``tools/memory_tool.py`` 的 ``MemoryStore`` 移植,适配 deerflow 的
|
||||
per-user / per-agent 目录结构与 :class:`MemoryProvider` 接口。
|
||||
|
||||
两类存储
|
||||
========
|
||||
- ``USER.md`` 用户画像 —— LLM 主动写入的 ``target="user"`` 记忆,跨 agent 共享。
|
||||
路径:``{base_dir}/users/{user_id}/USER.md``
|
||||
- ``MEMORY.md`` agent 工作笔记 —— LLM 主动写入的 ``target="memory"`` 记忆,agent 隔离。
|
||||
路径:``{base_dir}/users/{user_id}/agents/{agent_id}/MEMORY.md``
|
||||
|
||||
设计要点
|
||||
========
|
||||
- 条目以 ``\\n§\\n`` 分隔,支持多行;采用字符级配额 (与模型无关)。
|
||||
- frozen snapshot:``initialize`` 时读盘一次并冻结为系统提示词块,会话内写入
|
||||
直接落盘 (durable) 但不刷新快照 —— 保持系统提示词稳定,利于 prefix cache。
|
||||
- 读-改-写经文件锁串行化;写入用临时文件 + 原子 rename,避免并发读到半截文件。
|
||||
- 写入前经 :mod:`deerflow.agents.memory.security` 扫描注入/外渗。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.agents.memory.security import scan_memory_content
|
||||
from deerflow.config.memory_config import SecurityConfig
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 跨平台文件锁:Unix 用 fcntl,Windows 用 msvcrt。
|
||||
_fcntl: Any = None
|
||||
_msvcrt: Any = None
|
||||
try: # pragma: no cover - 平台相关
|
||||
import fcntl as _fcntl_mod
|
||||
|
||||
_fcntl = _fcntl_mod
|
||||
except ImportError: # pragma: no cover - Windows
|
||||
try:
|
||||
import msvcrt as _msvcrt_mod
|
||||
|
||||
_msvcrt = _msvcrt_mod
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
#: 条目分隔符。``§`` = section sign (U+00A7)。
|
||||
ENTRY_DELIMITER = "\n§\n"
|
||||
|
||||
#: target 取值。
|
||||
_VALID_TARGETS = ("memory", "user")
|
||||
|
||||
# 同一进程内对同一文件路径的写入串行化锁 (文件锁之上再加一层,降低自争用)。
|
||||
_PATH_LOCKS: dict[str, threading.Lock] = {}
|
||||
_PATH_LOCKS_GUARD = threading.Lock()
|
||||
|
||||
|
||||
def _get_path_lock(path: Path) -> threading.Lock:
|
||||
"""返回某文件路径对应的进程内锁 (按路径字符串缓存)。"""
|
||||
key = str(path)
|
||||
with _PATH_LOCKS_GUARD:
|
||||
lock = _PATH_LOCKS.get(key)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
_PATH_LOCKS[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _file_lock(path: Path):
|
||||
"""获取某文件的排他锁,保证读-改-写安全。
|
||||
|
||||
用单独的 ``.lock`` 文件加锁,这样记忆文件本身仍可被 ``os.replace`` 原子替换。
|
||||
进程内先抢 :func:`_get_path_lock`,再抢跨进程文件锁。
|
||||
"""
|
||||
process_lock = _get_path_lock(path)
|
||||
process_lock.acquire()
|
||||
lock_path = path.with_suffix(path.suffix + ".lock")
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if _fcntl is None and _msvcrt is None: # pragma: no cover - 无锁平台
|
||||
yield
|
||||
return
|
||||
if _msvcrt is not None and (not lock_path.exists() or lock_path.stat().st_size == 0):
|
||||
lock_path.write_text(" ", encoding="utf-8")
|
||||
fd = open(lock_path, "r+" if _msvcrt is not None else "a+")
|
||||
try:
|
||||
if _fcntl is not None:
|
||||
_fcntl.flock(fd, _fcntl.LOCK_EX)
|
||||
else: # pragma: no cover - Windows
|
||||
fd.seek(0)
|
||||
_msvcrt.locking(fd.fileno(), _msvcrt.LK_LOCK, 1)
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
if _fcntl is not None:
|
||||
_fcntl.flock(fd, _fcntl.LOCK_UN)
|
||||
elif _msvcrt is not None: # pragma: no cover - Windows
|
||||
fd.seek(0)
|
||||
_msvcrt.locking(fd.fileno(), _msvcrt.LK_UNLCK, 1)
|
||||
except OSError:
|
||||
pass
|
||||
fd.close()
|
||||
finally:
|
||||
process_lock.release()
|
||||
|
||||
|
||||
def _read_entries(path: Path) -> list[str]:
|
||||
"""读取一个记忆文件并按分隔符切成条目列表。
|
||||
|
||||
无需加锁:写入用原子 rename,读者总能看到旧文件或新文件的完整内容。
|
||||
"""
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return []
|
||||
if not raw.strip():
|
||||
return []
|
||||
entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)]
|
||||
return [e for e in entries if e]
|
||||
|
||||
|
||||
def _write_entries(path: Path, entries: list[str]) -> None:
|
||||
"""用临时文件 + 原子 rename 写入记忆文件。"""
|
||||
content = ENTRY_DELIMITER.join(entries) if entries else ""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".mem_")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class BuiltinFileProvider(MemoryProvider):
|
||||
"""本地文件记忆 provider,始终可用,不可移除。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
memory_char_limit: int = 2200,
|
||||
user_char_limit: int = 1375,
|
||||
deduplicate_on_load: bool = True,
|
||||
security: SecurityConfig | None = None,
|
||||
) -> None:
|
||||
self._memory_char_limit = memory_char_limit
|
||||
self._user_char_limit = user_char_limit
|
||||
self._deduplicate_on_load = deduplicate_on_load
|
||||
self._security = security or SecurityConfig()
|
||||
|
||||
self._user_id: str = DEFAULT_USER_ID
|
||||
self._agent_id: str = "default"
|
||||
self._user_path: Path | None = None
|
||||
self._memory_path: Path | None = None
|
||||
|
||||
# 实时状态 (工具写入后立即更新并落盘)
|
||||
self._user_entries: list[str] = []
|
||||
self._memory_entries: list[str] = []
|
||||
# frozen snapshot (initialize 时冻结,系统提示词注入用)
|
||||
self._snapshot: dict[str, str] = {"user": "", "memory": ""}
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "builtin"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""本地文件 provider 无外部依赖,始终可用。"""
|
||||
return True
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str = "",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""解析 per-user / per-agent 路径,加载文件并冻结系统提示词快照。"""
|
||||
self._user_id = user_id or DEFAULT_USER_ID
|
||||
self._agent_id = (agent_id or "default").lower()
|
||||
paths = get_paths()
|
||||
self._user_path = paths.user_profile_md_file(self._user_id)
|
||||
self._memory_path = paths.user_agent_memory_md_file(self._user_id, self._agent_id)
|
||||
self._load_from_disk()
|
||||
|
||||
def _load_from_disk(self) -> None:
|
||||
"""从磁盘加载两个文件,去重,并冻结系统提示词快照。"""
|
||||
assert self._user_path is not None and self._memory_path is not None
|
||||
self._user_entries = _read_entries(self._user_path)
|
||||
self._memory_entries = _read_entries(self._memory_path)
|
||||
if self._deduplicate_on_load:
|
||||
self._user_entries = list(dict.fromkeys(self._user_entries))
|
||||
self._memory_entries = list(dict.fromkeys(self._memory_entries))
|
||||
self._snapshot = {
|
||||
"user": self._render_block("user", self._user_entries),
|
||||
"memory": self._render_block("memory", self._memory_entries),
|
||||
}
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""返回 frozen snapshot:USER PROFILE 块 + MEMORY 块。
|
||||
|
||||
返回 ``initialize`` 时冻结的内容,会话内的工具写入不影响它 ——
|
||||
保持系统提示词稳定,利于 prefix cache。
|
||||
"""
|
||||
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
|
||||
|
||||
不经 MemoryManager 路由,因此这里返回空列表。
|
||||
"""
|
||||
return []
|
||||
|
||||
# ---- 工具写入操作 (供 memory 工具直接调用) ---------------------------
|
||||
|
||||
def add(self, target: str, content: str) -> dict[str, Any]:
|
||||
"""追加一条记忆条目。超出字符配额时返回错误。"""
|
||||
content = (content or "").strip()
|
||||
if not content:
|
||||
return {"success": False, "error": "条目内容不能为空。"}
|
||||
if (err := self._scan(content)) is not None:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
if content in entries:
|
||||
return self._success(target, "条目已存在,未重复添加。")
|
||||
limit = self._char_limit(target)
|
||||
new_total = len(ENTRY_DELIMITER.join([*entries, content]))
|
||||
if new_total > limit:
|
||||
current = self._char_count(entries)
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"记忆已用 {current:,}/{limit:,} 字符,"
|
||||
f"再加这条 ({len(content)} 字符) 会超出配额。"
|
||||
"请先 replace 或 remove 现有条目。"
|
||||
),
|
||||
"entries": entries,
|
||||
}
|
||||
entries.append(content)
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已添加。")
|
||||
|
||||
def replace(self, target: str, old_text: str, new_content: str) -> dict[str, Any]:
|
||||
"""用 ``old_text`` 短串定位条目,替换为 ``new_content``。"""
|
||||
old_text = (old_text or "").strip()
|
||||
new_content = (new_content or "").strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text 不能为空。"}
|
||||
if not new_content:
|
||||
return {"success": False, "error": "new_content 不能为空;删除条目请用 remove。"}
|
||||
if (err := self._scan(new_content)) is not None:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
if not matches:
|
||||
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
|
||||
if len({e for _, e in matches}) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
|
||||
"matches": previews,
|
||||
}
|
||||
idx = matches[0][0]
|
||||
limit = self._char_limit(target)
|
||||
test_entries = list(entries)
|
||||
test_entries[idx] = new_content
|
||||
new_total = len(ENTRY_DELIMITER.join(test_entries))
|
||||
if new_total > limit:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"替换后会达到 {new_total:,}/{limit:,} 字符,超出配额。"
|
||||
"请缩短新内容或先删除其他条目。"
|
||||
),
|
||||
}
|
||||
entries[idx] = new_content
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已替换。")
|
||||
|
||||
def remove(self, target: str, old_text: str) -> dict[str, Any]:
|
||||
"""用 ``old_text`` 短串定位并删除条目。"""
|
||||
old_text = (old_text or "").strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text 不能为空。"}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
if not matches:
|
||||
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
|
||||
if len({e for _, e in matches}) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
|
||||
"matches": previews,
|
||||
}
|
||||
entries.pop(matches[0][0])
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已删除。")
|
||||
|
||||
def read(self, target: str) -> list[str]:
|
||||
"""返回某 target 的实时条目列表 (从磁盘重读)。"""
|
||||
return self._reload_target(target)
|
||||
|
||||
def usage(self, target: str) -> str:
|
||||
"""返回某 target 的字符用量描述,如 ``'12% — 165/1375 字符'``。"""
|
||||
entries = self._reload_target(target)
|
||||
current = self._char_count(entries)
|
||||
limit = self._char_limit(target)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
return f"{pct}% — {current:,}/{limit:,} 字符"
|
||||
|
||||
# ---- 内部辅助 --------------------------------------------------------
|
||||
|
||||
def _scan(self, content: str) -> str | None:
|
||||
return scan_memory_content(
|
||||
content,
|
||||
scan_threats=self._security.scan_content,
|
||||
block_invisible_unicode=self._security.block_invisible_unicode,
|
||||
)
|
||||
|
||||
def _path_for(self, target: str) -> Path:
|
||||
if target not in _VALID_TARGETS:
|
||||
raise ValueError(f"非法 target '{target}',只能是 'memory' 或 'user'。")
|
||||
if self._user_path is None or self._memory_path is None:
|
||||
raise RuntimeError("BuiltinFileProvider 未初始化,请先调用 initialize()。")
|
||||
return self._user_path if target == "user" else self._memory_path
|
||||
|
||||
def _char_limit(self, target: str) -> int:
|
||||
return self._user_char_limit if target == "user" else self._memory_char_limit
|
||||
|
||||
@staticmethod
|
||||
def _char_count(entries: list[str]) -> int:
|
||||
return len(ENTRY_DELIMITER.join(entries)) if entries else 0
|
||||
|
||||
def _reload_target(self, target: str) -> list[str]:
|
||||
"""加锁状态下从磁盘重读条目 (拿到其他会话写入的最新状态)。"""
|
||||
entries = _read_entries(self._path_for(target))
|
||||
if self._deduplicate_on_load:
|
||||
entries = list(dict.fromkeys(entries))
|
||||
self._set_entries(target, entries)
|
||||
return entries
|
||||
|
||||
def _set_entries(self, target: str, entries: list[str]) -> None:
|
||||
if target == "user":
|
||||
self._user_entries = entries
|
||||
else:
|
||||
self._memory_entries = entries
|
||||
|
||||
def _success(self, target: str, message: str) -> dict[str, Any]:
|
||||
entries = self._user_entries if target == "user" else self._memory_entries
|
||||
current = self._char_count(entries)
|
||||
limit = self._char_limit(target)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
return {
|
||||
"success": True,
|
||||
"target": target,
|
||||
"entries": entries,
|
||||
"entry_count": len(entries),
|
||||
"usage": f"{pct}% — {current:,}/{limit:,} 字符",
|
||||
"message": message,
|
||||
}
|
||||
|
||||
def _render_block(self, target: str, entries: list[str]) -> str:
|
||||
"""渲染一个系统提示词块 (带表头和用量指示)。"""
|
||||
if not entries:
|
||||
return ""
|
||||
limit = self._char_limit(target)
|
||||
content = ENTRY_DELIMITER.join(entries)
|
||||
current = len(content)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
if target == "user":
|
||||
header = f"用户画像 USER PROFILE [{pct}% — {current:,}/{limit:,} 字符]"
|
||||
else:
|
||||
header = f"记忆笔记 MEMORY [{pct}% — {current:,}/{limit:,} 字符]"
|
||||
separator = "═" * 46
|
||||
return f"{separator}\n{header}\n{separator}\n{content}"
|
||||
|
||||
|
||||
__all__ = ["BuiltinFileProvider", "ENTRY_DELIMITER"]
|
||||
+462
@@ -0,0 +1,462 @@
|
||||
"""Memory V2 - Hindsight 外部 provider。
|
||||
|
||||
Hindsight 是带知识图谱、实体消解、多策略检索的长期记忆服务。本 provider 以
|
||||
HTTP 客户端方式接入一个用户自起的 Hindsight Docker 实例 (``local_external`` 模式)。
|
||||
|
||||
双 bank 隔离 (B 方案,详见 docs/MEMORY_V2_DESIGN_ZH.md §3)
|
||||
==========================================================
|
||||
- 全局 user bank ``bank_id_template`` 例:``deerflow-user-{user_id}``
|
||||
存用户身份、偏好,跨 agent 共享。
|
||||
- agent 工作 bank ``work_bank_id_template`` 例:``deerflow-work-{user_id}-{agent_id}``
|
||||
存 agent 私有工作笔记。
|
||||
|
||||
写入路由:
|
||||
- LLM 主动 ``memory(target="user")`` -> 镜像到 user bank
|
||||
- LLM 主动 ``memory(target="memory")`` -> 镜像到 work bank
|
||||
- 自动 retain 的对话 (``sync_turn``) -> work bank
|
||||
|
||||
召回:``prefetch`` 并发查两个 bank,合并结果。
|
||||
|
||||
document_id 与轮次计数
|
||||
======================
|
||||
本 provider 被 ``build_memory_manager`` 每轮重新构造 —— 是无状态的。因此:
|
||||
- ``document_id`` 每次 ``initialize`` 用带时间戳的新值,每轮对话的 retain 落到
|
||||
各自独立的 Hindsight 文档。
|
||||
- ``retain_every_n_turns`` 的轮次判定依赖外部传入的 ``turn_index``
|
||||
(由 ``MemoryMiddleware`` 按对话历史算出);未传时退回进程内 ``_turn_counter``。
|
||||
|
||||
客户端生命周期
|
||||
==============
|
||||
``_run_coro`` 每次操作新建一个事件循环。aiohttp ``ClientSession`` 是
|
||||
loop-bound 的,跨 loop 复用会泄漏并失效。因此本 provider **不缓存客户端** ——
|
||||
每次记忆操作在 :meth:`_run_with_client` 里即开即关,保证 session 在其所属的
|
||||
loop 内被正常关闭。
|
||||
|
||||
降级
|
||||
====
|
||||
``hindsight-client`` 是可选依赖。未安装或未配置 ``api_url`` 时 :meth:`is_available`
|
||||
返回 ``False``,provider 不会被注册,记忆系统退回纯 builtin —— 不影响任何功能。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.config.memory_config import HindsightMemoryConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: hindsight-client 最低版本。
|
||||
MIN_CLIENT_VERSION = "0.4.22"
|
||||
|
||||
|
||||
def hindsight_client_available() -> bool:
|
||||
"""返回 ``hindsight-client`` 包是否已安装 (不导入实际客户端)。"""
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec("hindsight_client") is not None
|
||||
|
||||
|
||||
def _run_coro(coro: Any, timeout: float) -> Any:
|
||||
"""把一个 async coro 跑到完成并返回结果 (带超时)。
|
||||
|
||||
deerflow 的中间件钩子有同步也有异步路径;本 helper 两种场景都能用:
|
||||
- 当前线程没有运行中的事件循环 -> 直接 ``asyncio.run``
|
||||
- 当前线程已在事件循环里 -> 另起线程跑,避免嵌套循环报错
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if not in_loop:
|
||||
return asyncio.run(asyncio.wait_for(coro, timeout))
|
||||
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def _runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(asyncio.wait_for(coro, timeout))
|
||||
except BaseException as exc: # noqa: BLE001 - 跨线程传递异常
|
||||
box["error"] = exc
|
||||
|
||||
thread = threading.Thread(target=_runner, name="hindsight-coro", daemon=True)
|
||||
thread.start()
|
||||
thread.join(timeout + 5)
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
if "value" not in box:
|
||||
raise TimeoutError("Hindsight 操作超时")
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class HindsightProvider(MemoryProvider):
|
||||
"""Hindsight 外部记忆 provider。"""
|
||||
|
||||
def __init__(self, config: HindsightMemoryConfig | None = None) -> None:
|
||||
self._config = config or HindsightMemoryConfig()
|
||||
|
||||
self._user_id: str = ""
|
||||
self._agent_id: str = "default"
|
||||
self._thread_id: str = ""
|
||||
self._user_bank: str = ""
|
||||
self._work_bank: str = ""
|
||||
|
||||
# retain 批次的 document id 与轮次计数 (on_session_switch 会重置)
|
||||
self._document_id: str = ""
|
||||
self._turn_counter: int = 0
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "hindsight"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""配置完整且 hindsight-client 已安装时才可用。"""
|
||||
if not hindsight_client_available():
|
||||
logger.debug("Hindsight 不可用:hindsight-client 未安装")
|
||||
return False
|
||||
if not self._config.api_url:
|
||||
logger.debug("Hindsight 不可用:未配置 api_url")
|
||||
return False
|
||||
return True
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str = "",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""解析双 bank 名称,准备 retain 批次状态。客户端每次操作即开即关。"""
|
||||
self._user_id = user_id or "default"
|
||||
self._agent_id = (agent_id or "default").lower()
|
||||
self._thread_id = thread_id or ""
|
||||
self._user_bank = self._resolve_bank(self._config.bank_id_template)
|
||||
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
|
||||
self._reset_retain_batch()
|
||||
|
||||
def _resolve_bank(self, template: str) -> str:
|
||||
"""用 user_id / agent_id / thread_id 填充 bank 名称模板。
|
||||
|
||||
空占位符会塌缩 (例如 ``deerflow-{thread_id}`` 在无 thread 时变成 ``deerflow-``,
|
||||
再清理多余连字符)。
|
||||
"""
|
||||
name = template.format(
|
||||
user_id=self._user_id,
|
||||
agent_id=self._agent_id,
|
||||
thread_id=self._thread_id,
|
||||
)
|
||||
# 清理空占位符塌缩留下的多余连字符
|
||||
while "--" in name:
|
||||
name = name.replace("--", "-")
|
||||
return name.strip("-") or "deerflow"
|
||||
|
||||
def _reset_retain_batch(self) -> None:
|
||||
"""重置 retain 批次状态 (新的 document_id + 轮次清零)。
|
||||
|
||||
每次 ``initialize`` 用带时间戳的新 ``document_id`` —— 每轮对话的 retain
|
||||
落到各自独立的 Hindsight 文档。轮次计数清零,仅作为 ``turn_index`` 缺失
|
||||
时的退路。
|
||||
"""
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
self._document_id = f"{self._thread_id or 'thread'}-{ts}"
|
||||
self._turn_counter = 0
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""无需清理 —— 客户端在每次操作内即开即关,不缓存。"""
|
||||
|
||||
# ---- 客户端生命周期 --------------------------------------------------
|
||||
|
||||
def _client_factory(self) -> Any:
|
||||
"""创建一个新的 Hindsight HTTP 客户端。
|
||||
|
||||
每次记忆操作都新建一个、用完即关,**不缓存** —— aiohttp ClientSession
|
||||
是 loop-bound 的,而 ``_run_coro`` 每次新建事件循环。测试可覆盖本方法
|
||||
注入 fake 客户端。
|
||||
"""
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"base_url": self._config.api_url,
|
||||
"timeout": 30.0,
|
||||
}
|
||||
if self._config.api_key:
|
||||
kwargs["api_key"] = self._config.api_key
|
||||
return Hindsight(**kwargs)
|
||||
|
||||
@staticmethod
|
||||
async def _aclose_client(client: Any) -> None:
|
||||
"""关闭客户端 (兼容暴露 ``aclose`` 或 ``close`` 的实现)。"""
|
||||
close = getattr(client, "aclose", None) or getattr(client, "close", None)
|
||||
if close is None:
|
||||
return
|
||||
try:
|
||||
result = close()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception as exc: # noqa: BLE001 - 关闭失败不影响结果
|
||||
logger.debug("Hindsight 客户端关闭失败:%s", exc)
|
||||
|
||||
async def _run_with_client(self, op: Any) -> Any:
|
||||
"""在当前事件循环内创建客户端、执行 ``op(client)``、保证关闭客户端。
|
||||
|
||||
``op`` 是一个接收 client、返回 awaitable 的协程函数。无论成功、异常还是
|
||||
被取消 (超时),``finally`` 都会关闭客户端 —— 杜绝 ``Unclosed client session``。
|
||||
"""
|
||||
client = self._client_factory()
|
||||
try:
|
||||
return await op(client)
|
||||
finally:
|
||||
await self._aclose_client(client)
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""返回一小段静态说明 (告诉 LLM Hindsight 处于哪种模式)。"""
|
||||
mode = self._config.memory_mode
|
||||
if mode == "context":
|
||||
return "" # context 模式纯后台,无需告知 LLM
|
||||
if mode == "tools":
|
||||
return (
|
||||
"# Hindsight 长期记忆\n"
|
||||
"可用 hindsight_recall 检索、hindsight_reflect 综合、hindsight_retain 存储。"
|
||||
)
|
||||
return (
|
||||
"# Hindsight 长期记忆\n"
|
||||
"相关记忆会自动注入上下文;也可用 hindsight_recall / hindsight_reflect / "
|
||||
"hindsight_retain 主动检索与存储。"
|
||||
)
|
||||
|
||||
# ---- 召回 ------------------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""并发召回两个 bank 并合并结果。
|
||||
|
||||
``memory_mode='tools'`` 或 ``auto_recall=False`` 时不做自动召回。
|
||||
召回失败 (Hindsight 不可达等) 返回空串,不抛异常。
|
||||
"""
|
||||
if self._config.memory_mode == "tools":
|
||||
return ""
|
||||
if not self._config.auto_recall:
|
||||
return ""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return ""
|
||||
if self._config.recall_max_input_chars and len(query) > self._config.recall_max_input_chars:
|
||||
query = query[: self._config.recall_max_input_chars]
|
||||
|
||||
try:
|
||||
merged = self._recall_both(query)
|
||||
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
|
||||
logger.debug("Hindsight 召回失败 (非致命):%s", exc)
|
||||
return ""
|
||||
|
||||
if not merged:
|
||||
return ""
|
||||
lines = "\n".join(f"- {text}" for text in merged)
|
||||
return (
|
||||
"# Hindsight 记忆召回 (跨会话持久上下文)\n"
|
||||
"下面是与当前对话相关的历史记忆,可据此回答关于用户和过往会话的问题。\n\n"
|
||||
f"{lines}"
|
||||
)
|
||||
|
||||
def _recall_both(self, query: str) -> list[str]:
|
||||
"""并发召回 user + work 两个 bank,合并去重。一次操作共用一个客户端。"""
|
||||
|
||||
async def _op(client: Any) -> tuple[Any, Any]:
|
||||
return await asyncio.gather(
|
||||
client.arecall(
|
||||
bank_id=self._user_bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
max_tokens=self._config.recall_max_tokens,
|
||||
),
|
||||
client.arecall(
|
||||
bank_id=self._work_bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
max_tokens=self._config.recall_max_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
user_resp, work_resp = _run_coro(self._run_with_client(_op), timeout=15)
|
||||
return self._merge_results(
|
||||
self._extract_results(user_resp),
|
||||
self._extract_results(work_resp),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_results(resp: Any) -> list[tuple[float, str]]:
|
||||
"""从一个 recall 响应里抽出 ``[(score, text), ...]``。"""
|
||||
results = getattr(resp, "results", None) or []
|
||||
out: list[tuple[float, str]] = []
|
||||
for r in results:
|
||||
text = (getattr(r, "text", "") or "").strip()
|
||||
if text:
|
||||
out.append((float(getattr(r, "score", 0.0) or 0.0), text))
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _merge_results(
|
||||
user_results: list[tuple[float, str]],
|
||||
work_results: list[tuple[float, str]],
|
||||
) -> list[str]:
|
||||
"""合并双 bank 召回结果:按 score 倒序,去重保序。"""
|
||||
combined = sorted(user_results + work_results, key=lambda x: -x[0])
|
||||
seen: set[str] = set()
|
||||
merged: list[str] = []
|
||||
for _, text in combined:
|
||||
if text not in seen:
|
||||
seen.add(text)
|
||||
merged.append(text)
|
||||
return merged
|
||||
|
||||
# ---- 写入 ------------------------------------------------------------
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""自动 retain 一轮对话到 work bank。
|
||||
|
||||
``auto_retain=False`` 时不写。按 ``retain_every_n_turns`` 控制频率。
|
||||
|
||||
轮次判定优先用调用方传入的 ``turn_index`` (由 MemoryMiddleware 按对话历史
|
||||
算出) —— 因为本 provider 每轮被重建、进程内 ``_turn_counter`` 会被重置,
|
||||
单靠它在 ``retain_every_n_turns > 1`` 时会永远不触发。``turn_index`` 缺失
|
||||
时才退回 ``_turn_counter``。
|
||||
"""
|
||||
if not self._config.auto_retain:
|
||||
return
|
||||
|
||||
if turn_index is not None:
|
||||
counter = turn_index
|
||||
else:
|
||||
self._turn_counter += 1
|
||||
counter = self._turn_counter
|
||||
|
||||
every = max(1, self._config.retain_every_n_turns)
|
||||
if counter % every != 0:
|
||||
return
|
||||
|
||||
turn_text = (
|
||||
f"{self._config.retain_user_prefix}: {user_content}\n"
|
||||
f"{self._config.retain_assistant_prefix}: {assistant_content}"
|
||||
)
|
||||
try:
|
||||
self._retain(self._work_bank, turn_text, kind="conversation")
|
||||
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
|
||||
logger.warning("Hindsight sync_turn 写入失败:%s", exc)
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入时镜像到 Hindsight。
|
||||
|
||||
``target="user"`` -> user bank;``target="memory"`` -> work bank。
|
||||
目前只镜像 ``add`` / ``replace`` 的新内容 (Hindsight 是 append-only 语义,
|
||||
``remove`` 不反向删除)。
|
||||
"""
|
||||
if action not in ("add", "replace"):
|
||||
return
|
||||
content = (content or "").strip()
|
||||
if not content:
|
||||
return
|
||||
bank = self._user_bank if target == "user" else self._work_bank
|
||||
try:
|
||||
self._retain(bank, content, kind=f"builtin_{target}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Hindsight on_memory_write 镜像失败:%s", exc)
|
||||
|
||||
def _retain(self, bank_id: str, content: str, *, kind: str) -> None:
|
||||
"""向某 bank 写入一条记忆。客户端即开即关。"""
|
||||
metadata = {
|
||||
"retained_at": _utc_now_iso(),
|
||||
"kind": kind,
|
||||
"user_id": self._user_id,
|
||||
"agent_id": self._agent_id,
|
||||
}
|
||||
if self._thread_id:
|
||||
metadata["thread_id"] = self._thread_id
|
||||
|
||||
async def _op(client: Any) -> Any:
|
||||
return await client.aretain(
|
||||
bank_id=bank_id,
|
||||
content=content,
|
||||
context=self._config.bank_retain_mission or None,
|
||||
metadata=metadata,
|
||||
document_id=self._document_id,
|
||||
retain_async=self._config.retain_async,
|
||||
)
|
||||
|
||||
_run_coro(self._run_with_client(_op), timeout=15)
|
||||
|
||||
def reflect(self, query: str, *, use_user_bank: bool = True) -> str:
|
||||
"""对某 bank 做跨记忆综合 (LLM 推理)。供 hindsight_reflect 工具调用。"""
|
||||
bank = self._user_bank if use_user_bank else self._work_bank
|
||||
|
||||
async def _op(client: Any) -> Any:
|
||||
return await client.areflect(
|
||||
bank_id=bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
)
|
||||
|
||||
resp = _run_coro(self._run_with_client(_op), timeout=30)
|
||||
return getattr(resp, "text", "") or ""
|
||||
|
||||
def recall(self, query: str) -> list[str]:
|
||||
"""召回双 bank 并返回文本列表。供 hindsight_recall 工具调用。"""
|
||||
return self._recall_both(query)
|
||||
|
||||
def retain_to_user_bank(self, content: str) -> None:
|
||||
"""主动写入 user bank。供 hindsight_retain 工具调用。"""
|
||||
self._retain(self._user_bank, content, kind="tool_retain")
|
||||
|
||||
# ---- LLM 工具 --------------------------------------------------------
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""Hindsight 的 3 个工具以 LangChain BaseTool 独立注册,不经 manager 路由。"""
|
||||
return []
|
||||
|
||||
# ---- 可选 hooks ------------------------------------------------------
|
||||
|
||||
def on_session_switch(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""thread 切换:重新解析 work bank,重置 retain 批次。"""
|
||||
self._thread_id = new_thread_id
|
||||
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
|
||||
self._reset_retain_batch()
|
||||
|
||||
|
||||
__all__ = ["HindsightProvider", "hindsight_client_available", "MIN_CLIENT_VERSION"]
|
||||
@@ -1,277 +0,0 @@
|
||||
"""Memory update queue with debounce mechanism."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationContext:
|
||||
"""Context for a conversation to be processed for memory update."""
|
||||
|
||||
thread_id: str
|
||||
messages: list[Any]
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
agent_name: str | None = None
|
||||
user_id: str | None = None
|
||||
correction_detected: bool = False
|
||||
reinforcement_detected: bool = False
|
||||
|
||||
|
||||
class MemoryUpdateQueue:
|
||||
"""Queue for memory updates with debounce mechanism.
|
||||
|
||||
This queue collects conversation contexts and processes them after
|
||||
a configurable debounce period. Multiple conversations received within
|
||||
the debounce window are batched together.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the memory update queue."""
|
||||
self._queue: list[ConversationContext] = []
|
||||
self._lock = threading.Lock()
|
||||
self._timer: threading.Timer | None = None
|
||||
self._processing = False
|
||||
|
||||
def add(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation to the update queue.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
messages: The conversation messages.
|
||||
agent_name: If provided, memory is stored per-agent. If None, uses global memory.
|
||||
user_id: The user ID captured at enqueue time. Stored in ConversationContext so it
|
||||
survives the threading.Timer boundary (ContextVar does not propagate across
|
||||
raw threads).
|
||||
correction_detected: Whether recent turns include an explicit correction signal.
|
||||
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
|
||||
"""
|
||||
config = get_memory_config()
|
||||
if not config.enabled:
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._reset_timer()
|
||||
|
||||
logger.info("Memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def add_nowait(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation and start processing immediately in the background."""
|
||||
config = get_memory_config()
|
||||
if not config.enabled:
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._schedule_timer(0)
|
||||
|
||||
logger.info("Memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def _enqueue_locked(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
user_id: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
) -> None:
|
||||
existing_context = next(
|
||||
(context for context in self._queue if context.thread_id == thread_id),
|
||||
None,
|
||||
)
|
||||
merged_correction_detected = correction_detected or (existing_context.correction_detected if existing_context is not None else False)
|
||||
merged_reinforcement_detected = reinforcement_detected or (existing_context.reinforcement_detected if existing_context is not None else False)
|
||||
context = ConversationContext(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=merged_correction_detected,
|
||||
reinforcement_detected=merged_reinforcement_detected,
|
||||
)
|
||||
|
||||
self._queue = [c for c in self._queue if c.thread_id != thread_id]
|
||||
self._queue.append(context)
|
||||
|
||||
def _reset_timer(self) -> None:
|
||||
"""Reset the debounce timer."""
|
||||
config = get_memory_config()
|
||||
self._schedule_timer(config.debounce_seconds)
|
||||
|
||||
logger.debug("Memory update timer set for %ss", config.debounce_seconds)
|
||||
|
||||
def _schedule_timer(self, delay_seconds: float) -> None:
|
||||
"""Schedule queue processing after the provided delay."""
|
||||
# Cancel existing timer if any
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
|
||||
self._timer = threading.Timer(
|
||||
delay_seconds,
|
||||
self._process_queue,
|
||||
)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _process_queue(self) -> None:
|
||||
"""Process all queued conversation contexts."""
|
||||
# Import here to avoid circular dependency
|
||||
from deerflow.agents.memory.updater import MemoryUpdater
|
||||
|
||||
with self._lock:
|
||||
if self._processing:
|
||||
# Preserve immediate flush semantics even if another worker is active.
|
||||
self._schedule_timer(0)
|
||||
return
|
||||
|
||||
if not self._queue:
|
||||
return
|
||||
|
||||
self._processing = True
|
||||
contexts_to_process = self._queue.copy()
|
||||
self._queue.clear()
|
||||
self._timer = None
|
||||
|
||||
logger.info("Processing %d queued memory updates", len(contexts_to_process))
|
||||
|
||||
try:
|
||||
updater = MemoryUpdater()
|
||||
|
||||
for context in contexts_to_process:
|
||||
try:
|
||||
logger.info("Updating memory for thread %s", context.thread_id)
|
||||
success = updater.update_memory(
|
||||
messages=context.messages,
|
||||
thread_id=context.thread_id,
|
||||
agent_name=context.agent_name,
|
||||
correction_detected=context.correction_detected,
|
||||
reinforcement_detected=context.reinforcement_detected,
|
||||
user_id=context.user_id,
|
||||
)
|
||||
if success:
|
||||
logger.info("Memory updated successfully for thread %s", context.thread_id)
|
||||
else:
|
||||
logger.warning("Memory update skipped/failed for thread %s", context.thread_id)
|
||||
except Exception as e:
|
||||
logger.error("Error updating memory for thread %s: %s", context.thread_id, e)
|
||||
|
||||
# Small delay between updates to avoid rate limiting
|
||||
if len(contexts_to_process) > 1:
|
||||
time.sleep(0.5)
|
||||
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processing = False
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force immediate processing of the queue.
|
||||
|
||||
This is useful for testing or graceful shutdown.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
|
||||
self._process_queue()
|
||||
|
||||
def flush_nowait(self) -> None:
|
||||
"""Start queue processing immediately in a background thread."""
|
||||
with self._lock:
|
||||
# Daemon thread: queued messages may be lost if the process exits
|
||||
# before _process_queue completes. Acceptable for best-effort memory updates.
|
||||
self._schedule_timer(0)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear the queue without processing.
|
||||
|
||||
This is useful for testing.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._queue.clear()
|
||||
self._processing = False
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
"""Get the number of pending updates."""
|
||||
with self._lock:
|
||||
return len(self._queue)
|
||||
|
||||
@property
|
||||
def is_processing(self) -> bool:
|
||||
"""Check if the queue is currently being processed."""
|
||||
with self._lock:
|
||||
return self._processing
|
||||
|
||||
|
||||
# Global singleton instance
|
||||
_memory_queue: MemoryUpdateQueue | None = None
|
||||
_queue_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_queue() -> MemoryUpdateQueue:
|
||||
"""Get the global memory update queue singleton.
|
||||
|
||||
Returns:
|
||||
The memory update queue instance.
|
||||
"""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is None:
|
||||
_memory_queue = MemoryUpdateQueue()
|
||||
return _memory_queue
|
||||
|
||||
|
||||
def reset_memory_queue() -> None:
|
||||
"""Reset the global memory queue.
|
||||
|
||||
This is useful for testing.
|
||||
"""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is not None:
|
||||
_memory_queue.clear()
|
||||
_memory_queue = None
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Memory V2 - 记忆内容安全扫描。
|
||||
|
||||
记忆条目会被注入到系统提示词里,因此写入前必须扫描,阻断:
|
||||
- Prompt 注入 / 角色劫持 / 隐藏指令
|
||||
- 凭证外渗 (curl/wget 带密钥、读 .env 等敏感文件)
|
||||
- 持久化后门 (写 authorized_keys 等)
|
||||
- 不可见 unicode (零宽字符、双向控制符)
|
||||
|
||||
这是从 Hermes ``tools/memory_tool.py`` 的 ``_scan_memory_content`` 移植而来,
|
||||
并针对 deerflow 的目录结构调整了敏感路径模式。
|
||||
|
||||
扫描默认开启,可由 ``config.memory.security.scan_content`` /
|
||||
``block_invisible_unicode`` 关闭(不建议)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 威胁正则模式表:(正则, 模式 id)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_THREAT_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
# --- Prompt 注入 / 角色劫持 ---
|
||||
# 允许多个限定词,如 "ignore ALL above instructions"
|
||||
(re.compile(r"ignore\s+(?:(?:previous|all|above|prior|any)\s+)+instructions", re.IGNORECASE), "prompt_injection"),
|
||||
(re.compile(r"忽略(?:以上|之前|上述|所有|全部|一切)+(?:的)?(?:指令|指示|要求|提示词)"), "prompt_injection_zh"),
|
||||
(re.compile(r"you\s+are\s+now\s+", re.IGNORECASE), "role_hijack"),
|
||||
(re.compile(r"do\s+not\s+tell\s+the\s+user", re.IGNORECASE), "deception_hide"),
|
||||
(re.compile(r"不要(告诉|让)用户(知道)?"), "deception_hide_zh"),
|
||||
(re.compile(r"system\s+prompt\s+override", re.IGNORECASE), "sys_prompt_override"),
|
||||
(re.compile(r"disregard\s+(your|all|any)\s+(instructions|rules|guidelines)", re.IGNORECASE), "disregard_rules"),
|
||||
(re.compile(r"act\s+as\s+(if|though)\s+you\s+(have\s+no|don'?t\s+have)\s+(restrictions|limits|rules)", re.IGNORECASE), "bypass_restrictions"),
|
||||
# --- 凭证外渗 ---
|
||||
(re.compile(r"curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_curl"),
|
||||
(re.compile(r"wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_wget"),
|
||||
(re.compile(r"cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)", re.IGNORECASE), "read_secrets"),
|
||||
# --- 持久化后门 ---
|
||||
(re.compile(r"authorized_keys", re.IGNORECASE), "ssh_backdoor"),
|
||||
(re.compile(r"(\$HOME|~)/\.ssh", re.IGNORECASE), "ssh_access"),
|
||||
]
|
||||
|
||||
# 不可见 unicode 字符:零宽字符 + 双向控制符。
|
||||
# 它们肉眼不可见,常被用来在记忆里藏注入 payload。
|
||||
_INVISIBLE_CHARS: frozenset[str] = frozenset(
|
||||
{
|
||||
"", # ZERO WIDTH SPACE
|
||||
"", # ZERO WIDTH NON-JOINER
|
||||
"", # ZERO WIDTH JOINER
|
||||
"", # WORD JOINER
|
||||
"", # ZERO WIDTH NO-BREAK SPACE / BOM
|
||||
"", # LEFT-TO-RIGHT EMBEDDING
|
||||
"", # RIGHT-TO-LEFT EMBEDDING
|
||||
"", # POP DIRECTIONAL FORMATTING
|
||||
"", # LEFT-TO-RIGHT OVERRIDE
|
||||
"", # RIGHT-TO-LEFT OVERRIDE
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class MemoryContentBlocked(ValueError):
|
||||
"""记忆内容被安全扫描阻断时抛出。
|
||||
|
||||
``pattern_id`` 标明命中的威胁模式,便于上层返回结构化错误。
|
||||
"""
|
||||
|
||||
def __init__(self, pattern_id: str, message: str) -> None:
|
||||
self.pattern_id = pattern_id
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def scan_memory_content(
|
||||
content: str,
|
||||
*,
|
||||
scan_threats: bool = True,
|
||||
block_invisible_unicode: bool = True,
|
||||
) -> str | None:
|
||||
"""扫描记忆内容,命中威胁时返回错误描述,干净则返回 ``None``。
|
||||
|
||||
不抛异常 —— 返回字符串方便工具层直接塞进 ``{"success": False, "error": ...}``。
|
||||
需要异常语义时用 :func:`assert_memory_content_safe`。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
content:
|
||||
待写入的记忆条目内容。
|
||||
scan_threats:
|
||||
是否扫描注入/外渗模式。对应 ``config.memory.security.scan_content``。
|
||||
block_invisible_unicode:
|
||||
是否阻断不可见 unicode。对应 ``config.memory.security.block_invisible_unicode``。
|
||||
"""
|
||||
if block_invisible_unicode:
|
||||
for char in _INVISIBLE_CHARS:
|
||||
if char in content:
|
||||
return (
|
||||
f"已阻断:内容包含不可见 unicode 字符 U+{ord(char):04X},"
|
||||
"可能是注入 payload。"
|
||||
)
|
||||
|
||||
if scan_threats:
|
||||
for pattern, pattern_id in _THREAT_PATTERNS:
|
||||
if pattern.search(content):
|
||||
return (
|
||||
f"已阻断:内容命中威胁模式 '{pattern_id}'。"
|
||||
"记忆条目会被注入系统提示词,不允许包含注入或外渗内容。"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def assert_memory_content_safe(
|
||||
content: str,
|
||||
*,
|
||||
scan_threats: bool = True,
|
||||
block_invisible_unicode: bool = True,
|
||||
) -> None:
|
||||
"""扫描记忆内容,命中威胁时抛出 :class:`MemoryContentBlocked`。
|
||||
|
||||
:func:`scan_memory_content` 的异常版本,供需要异常语义的调用方使用。
|
||||
"""
|
||||
error = scan_memory_content(
|
||||
content,
|
||||
scan_threats=scan_threats,
|
||||
block_invisible_unicode=block_invisible_unicode,
|
||||
)
|
||||
if error is not None:
|
||||
# 从错误描述里反推 pattern_id 不可靠,这里重新扫一遍拿确切 id。
|
||||
pattern_id = "invisible_unicode"
|
||||
for pattern, pid in _THREAT_PATTERNS:
|
||||
if pattern.search(content):
|
||||
pattern_id = pid
|
||||
break
|
||||
raise MemoryContentBlocked(pattern_id, error)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryContentBlocked",
|
||||
"scan_memory_content",
|
||||
"assert_memory_content_safe",
|
||||
]
|
||||
@@ -1,231 +0,0 @@
|
||||
"""Memory storage providers."""
|
||||
|
||||
import abc
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def utc_now_iso_z() -> str:
|
||||
"""Current UTC time as ISO-8601 with ``Z`` suffix (matches prior naive-UTC output)."""
|
||||
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
|
||||
|
||||
|
||||
def create_empty_memory() -> dict[str, Any]:
|
||||
"""Create an empty memory structure."""
|
||||
return {
|
||||
"version": "1.0",
|
||||
"lastUpdated": utc_now_iso_z(),
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"facts": [],
|
||||
}
|
||||
|
||||
|
||||
class MemoryStorage(abc.ABC):
|
||||
"""Abstract base class for memory storage providers."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data for the given agent."""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Force reload memory data for the given agent."""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Save memory data for the given agent."""
|
||||
pass
|
||||
|
||||
|
||||
class FileMemoryStorage(MemoryStorage):
|
||||
"""File-based memory storage provider."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the file memory storage."""
|
||||
# Per-user/agent memory cache: keyed by (user_id, agent_name) tuple (None = global)
|
||||
# Value: (memory_data, file_mtime)
|
||||
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
|
||||
# Guards all reads and writes to _memory_cache across concurrent callers.
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
def _validate_agent_name(self, agent_name: str) -> None:
|
||||
"""Validate that the agent name is safe to use in filesystem paths.
|
||||
|
||||
Uses the repository's established AGENT_NAME_PATTERN to ensure consistency
|
||||
across the codebase and prevent path traversal or other problematic characters.
|
||||
"""
|
||||
if not agent_name:
|
||||
raise ValueError("Agent name must be a non-empty string.")
|
||||
if not AGENT_NAME_PATTERN.match(agent_name):
|
||||
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
|
||||
|
||||
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
|
||||
"""Get the path to the memory file."""
|
||||
if user_id is not None:
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return get_paths().user_agent_memory_file(user_id, agent_name)
|
||||
config = get_memory_config()
|
||||
if config.storage_path and Path(config.storage_path).is_absolute():
|
||||
return Path(config.storage_path)
|
||||
return get_paths().user_memory_file(user_id)
|
||||
# Legacy: no user_id
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return get_paths().agent_memory_file(agent_name)
|
||||
config = get_memory_config()
|
||||
if config.storage_path:
|
||||
p = Path(config.storage_path)
|
||||
return p if p.is_absolute() else get_paths().base_dir / p
|
||||
return get_paths().memory_file
|
||||
|
||||
def _load_memory_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data from file."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
|
||||
if not file_path.exists():
|
||||
return create_empty_memory()
|
||||
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning("Failed to load memory file: %s", e)
|
||||
return create_empty_memory()
|
||||
|
||||
@staticmethod
|
||||
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
|
||||
return (user_id, agent_name)
|
||||
|
||||
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data (cached with file modification time check)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
current_mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
cached = self._memory_cache.get(cache_key)
|
||||
if cached is not None and cached[1] == current_mtime:
|
||||
return cached[0]
|
||||
|
||||
memory_data = self._load_memory_from_file(agent_name, user_id=user_id)
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, current_mtime)
|
||||
|
||||
return memory_data
|
||||
|
||||
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Reload memory data from file, forcing cache invalidation."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
memory_data = self._load_memory_from_file(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
return memory_data
|
||||
|
||||
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Save memory data to file and update cache."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Shallow-copy before adding lastUpdated so the caller's dict is not
|
||||
# mutated as a side-effect, and the cache reference is not silently
|
||||
# updated before the file write succeeds.
|
||||
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
|
||||
|
||||
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(memory_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
temp_path.replace(file_path)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
logger.info("Memory saved to %s", file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error("Failed to save memory file: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
_storage_instance: MemoryStorage | None = None
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_storage() -> MemoryStorage:
|
||||
"""Get the configured memory storage instance."""
|
||||
global _storage_instance
|
||||
if _storage_instance is not None:
|
||||
return _storage_instance
|
||||
|
||||
with _storage_lock:
|
||||
if _storage_instance is not None:
|
||||
return _storage_instance
|
||||
|
||||
config = get_memory_config()
|
||||
storage_class_path = config.storage_class
|
||||
|
||||
try:
|
||||
module_path, class_name = storage_class_path.rsplit(".", 1)
|
||||
import importlib
|
||||
|
||||
module = importlib.import_module(module_path)
|
||||
storage_class = getattr(module, class_name)
|
||||
|
||||
# Validate that the configured storage is a MemoryStorage implementation
|
||||
if not isinstance(storage_class, type):
|
||||
raise TypeError(f"Configured memory storage '{storage_class_path}' is not a class: {storage_class!r}")
|
||||
if not issubclass(storage_class, MemoryStorage):
|
||||
raise TypeError(f"Configured memory storage '{storage_class_path}' is not a subclass of MemoryStorage")
|
||||
|
||||
_storage_instance = storage_class()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to load memory storage %s, falling back to FileMemoryStorage: %s",
|
||||
storage_class_path,
|
||||
e,
|
||||
)
|
||||
_storage_instance = FileMemoryStorage()
|
||||
|
||||
return _storage_instance
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
"""Hooks fired before summarization removes messages from state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
|
||||
from deerflow.agents.memory.queue import get_memory_queue
|
||||
from deerflow.agents.middlewares.summarization_middleware import SummarizationEvent
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
|
||||
|
||||
def memory_flush_hook(event: SummarizationEvent) -> None:
|
||||
"""Flush messages about to be summarized into the memory queue."""
|
||||
if not get_memory_config().enabled or not event.thread_id:
|
||||
return
|
||||
|
||||
filtered_messages = filter_messages_for_memory(list(event.messages_to_summarize))
|
||||
user_messages = [message for message in filtered_messages if getattr(message, "type", None) == "human"]
|
||||
assistant_messages = [message for message in filtered_messages if getattr(message, "type", None) == "ai"]
|
||||
if not user_messages or not assistant_messages:
|
||||
return
|
||||
|
||||
correction_detected = detect_correction(filtered_messages)
|
||||
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
|
||||
queue = get_memory_queue()
|
||||
queue.add_nowait(
|
||||
thread_id=event.thread_id,
|
||||
messages=filtered_messages,
|
||||
agent_name=event.agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
@@ -1,723 +0,0 @@
|
||||
"""Memory updater for reading, writing, and updating memory data."""
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.prompt import (
|
||||
MEMORY_UPDATE_PROMPT,
|
||||
format_conversation_for_update,
|
||||
)
|
||||
from deerflow.agents.memory.storage import (
|
||||
create_empty_memory,
|
||||
get_memory_storage,
|
||||
utc_now_iso_z,
|
||||
)
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Thread pool for offloading sync memory updates when called from an async
|
||||
# context. Unlike the previous asyncio.run() approach, this runs *sync*
|
||||
# model.invoke() calls — no event loop is created, so the langchain async
|
||||
# httpx client pool (globally cached via @lru_cache) is never touched and
|
||||
# cross-loop connection reuse is impossible.
|
||||
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=4,
|
||||
thread_name_prefix="memory-updater-sync",
|
||||
)
|
||||
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
|
||||
|
||||
|
||||
def _create_empty_memory() -> dict[str, Any]:
|
||||
"""Backward-compatible wrapper around the storage-layer empty-memory factory."""
|
||||
return create_empty_memory()
|
||||
|
||||
|
||||
def _save_memory_to_file(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Backward-compatible wrapper around the configured memory storage save path."""
|
||||
return get_memory_storage().save(memory_data, agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Get the current memory data via storage provider."""
|
||||
return get_memory_storage().load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Reload memory data via storage provider."""
|
||||
return get_memory_storage().reload(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Persist imported memory data via storage provider.
|
||||
|
||||
Args:
|
||||
memory_data: Full memory payload to persist.
|
||||
agent_name: If provided, imports into per-agent memory.
|
||||
user_id: If provided, scopes memory to a specific user.
|
||||
|
||||
Returns:
|
||||
The saved memory data after storage normalization.
|
||||
|
||||
Raises:
|
||||
OSError: If persisting the imported memory fails.
|
||||
"""
|
||||
storage = get_memory_storage()
|
||||
if not storage.save(memory_data, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save imported memory data")
|
||||
return storage.load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Clear all stored memory data and persist an empty structure."""
|
||||
cleared_memory = create_empty_memory()
|
||||
if not _save_memory_to_file(cleared_memory, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save cleared memory data")
|
||||
return cleared_memory
|
||||
|
||||
|
||||
def _validate_confidence(confidence: float) -> float:
|
||||
"""Validate persisted fact confidence so stored JSON stays standards-compliant."""
|
||||
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
|
||||
raise ValueError("confidence")
|
||||
return confidence
|
||||
|
||||
|
||||
def create_memory_fact(
|
||||
content: str,
|
||||
category: str = "context",
|
||||
confidence: float = 0.5,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new fact and persist the updated memory data."""
|
||||
normalized_content = content.strip()
|
||||
if not normalized_content:
|
||||
raise ValueError("content")
|
||||
|
||||
normalized_category = category.strip() or "context"
|
||||
validated_confidence = _validate_confidence(confidence)
|
||||
now = utc_now_iso_z()
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
facts = list(memory_data.get("facts", []))
|
||||
facts.append(
|
||||
{
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": normalized_category,
|
||||
"confidence": validated_confidence,
|
||||
"createdAt": now,
|
||||
"source": "manual",
|
||||
}
|
||||
)
|
||||
updated_memory["facts"] = facts
|
||||
|
||||
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save memory data after creating fact")
|
||||
|
||||
return updated_memory
|
||||
|
||||
|
||||
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Delete a fact by its id and persist the updated memory data."""
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
facts = memory_data.get("facts", [])
|
||||
updated_facts = [fact for fact in facts if fact.get("id") != fact_id]
|
||||
if len(updated_facts) == len(facts):
|
||||
raise KeyError(fact_id)
|
||||
|
||||
updated_memory = dict(memory_data)
|
||||
updated_memory["facts"] = updated_facts
|
||||
|
||||
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
|
||||
|
||||
return updated_memory
|
||||
|
||||
|
||||
def update_memory_fact(
|
||||
fact_id: str,
|
||||
content: str | None = None,
|
||||
category: str | None = None,
|
||||
confidence: float | None = None,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing fact and persist the updated memory data."""
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_facts: list[dict[str, Any]] = []
|
||||
found = False
|
||||
|
||||
for fact in memory_data.get("facts", []):
|
||||
if fact.get("id") == fact_id:
|
||||
found = True
|
||||
updated_fact = dict(fact)
|
||||
if content is not None:
|
||||
normalized_content = content.strip()
|
||||
if not normalized_content:
|
||||
raise ValueError("content")
|
||||
updated_fact["content"] = normalized_content
|
||||
if category is not None:
|
||||
updated_fact["category"] = category.strip() or "context"
|
||||
if confidence is not None:
|
||||
updated_fact["confidence"] = _validate_confidence(confidence)
|
||||
updated_facts.append(updated_fact)
|
||||
else:
|
||||
updated_facts.append(fact)
|
||||
|
||||
if not found:
|
||||
raise KeyError(fact_id)
|
||||
|
||||
updated_memory["facts"] = updated_facts
|
||||
|
||||
if not _save_memory_to_file(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
|
||||
|
||||
return updated_memory
|
||||
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
"""Extract plain text from LLM response content (str or list of content blocks).
|
||||
|
||||
Modern LLMs may return structured content as a list of blocks instead of a
|
||||
plain string, e.g. [{"type": "text", "text": "..."}]. Using str() on such
|
||||
content produces Python repr instead of the actual text, breaking JSON
|
||||
parsing downstream.
|
||||
|
||||
String chunks are concatenated without separators to avoid corrupting
|
||||
chunked JSON/text payloads. Dict-based text blocks are treated as full text
|
||||
blocks and joined with newlines for readability.
|
||||
"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
pieces: list[str] = []
|
||||
pending_str_parts: list[str] = []
|
||||
|
||||
def flush_pending_str_parts() -> None:
|
||||
if pending_str_parts:
|
||||
pieces.append("".join(pending_str_parts))
|
||||
pending_str_parts.clear()
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
pending_str_parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
flush_pending_str_parts()
|
||||
text_val = block.get("text")
|
||||
if isinstance(text_val, str):
|
||||
pieces.append(text_val)
|
||||
|
||||
flush_pending_str_parts()
|
||||
return "\n".join(pieces)
|
||||
return str(content)
|
||||
|
||||
|
||||
# Matches sentences that describe a file-upload *event* rather than general
|
||||
# file-related work. Deliberately narrow to avoid removing legitimate facts
|
||||
# such as "User works with CSV files" or "prefers PDF export".
|
||||
_UPLOAD_SENTENCE_RE = re.compile(
|
||||
r"[^.!?]*\b(?:"
|
||||
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
|
||||
r"|file\s+upload"
|
||||
r"|/mnt/user-data/uploads/"
|
||||
r"|<uploaded_files>"
|
||||
r")[^.!?]*[.!?]?\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Remove sentences about file uploads from all memory summaries and facts.
|
||||
|
||||
Uploaded files are session-scoped; persisting upload events in long-term
|
||||
memory causes the agent to search for non-existent files in future sessions.
|
||||
"""
|
||||
# Scrub summaries in user/history sections
|
||||
for section in ("user", "history"):
|
||||
section_data = memory_data.get(section, {})
|
||||
for _key, val in section_data.items():
|
||||
if isinstance(val, dict) and "summary" in val:
|
||||
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
|
||||
cleaned = re.sub(r" +", " ", cleaned)
|
||||
val["summary"] = cleaned
|
||||
|
||||
# Also remove any facts that describe upload events
|
||||
facts = memory_data.get("facts", [])
|
||||
if facts:
|
||||
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
|
||||
|
||||
return memory_data
|
||||
|
||||
|
||||
def _fact_content_key(content: Any) -> str | None:
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
stripped = content.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
return stripped.casefold()
|
||||
|
||||
|
||||
# Match a fenced code block, optionally with an info string (e.g. ```json).
|
||||
# We deliberately use a non-greedy DOTALL pattern so that *only* the contents
|
||||
# are kept — no leading prose like "Here is the updated memory:" and no
|
||||
# trailing chatter that would break ``json.loads``.
|
||||
_FENCED_BLOCK_RE = re.compile(r"```[a-zA-Z0-9_-]*\s*\n?(.*?)```", re.DOTALL)
|
||||
|
||||
# Match common "thinking" wrappers that some Chinese-domestic LLMs emit even
|
||||
# when the system prompt asks for JSON only (GLM, Qwen, MindIE-served models,
|
||||
# and others). Stripping these *before* JSON extraction prevents the body of
|
||||
# the reasoning text from being mistakenly parsed as the payload.
|
||||
_THINK_BLOCK_RE = re.compile(
|
||||
r"<\s*(think|thinking|reasoning|analysis)\s*>.*?<\s*/\s*\1\s*>",
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _extract_first_json_object(text: str) -> str | None:
|
||||
"""Return the substring spanning the first balanced ``{...}`` JSON object.
|
||||
|
||||
Walks the text once tracking brace depth, ignoring braces that appear
|
||||
inside double-quoted strings (with backslash-escape awareness). This is
|
||||
sufficient for the JSON dialect the prompt asks for; we deliberately do
|
||||
not handle single-quoted "JSON" because that would silently accept
|
||||
invalid responses we'd rather reject and retry on.
|
||||
"""
|
||||
start = text.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
|
||||
depth = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
for index in range(start, len(text)):
|
||||
char = text[index]
|
||||
if in_string:
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if char == "\\":
|
||||
escape = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
continue
|
||||
if char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start : index + 1]
|
||||
return None
|
||||
|
||||
|
||||
def _parse_memory_update_response(response_text: str) -> dict[str, Any]:
|
||||
"""Best-effort extraction of the JSON payload from a memory-update response.
|
||||
|
||||
Models served by ``vllm`` / ``MindIE`` (e.g. the GLM_5 endpoint at
|
||||
``http://10.106.57.108:8010/GLM_5/v1/chat/completions`` users have hit
|
||||
in production) routinely:
|
||||
|
||||
1. Wrap the JSON in a fenced code block (\\`\\`\\`json ... \\`\\`\\`)
|
||||
even though the prompt forbids it.
|
||||
2. Emit a ``<think>...</think>`` reasoning block before the answer.
|
||||
3. Add prose ("Here is the updated memory:") before/after the JSON.
|
||||
4. Return an empty body when the upstream model produced only reasoning
|
||||
and no answer (the original cause of "Expecting value: line 1
|
||||
column 1 (char 0)" reports).
|
||||
|
||||
This helper normalizes all four cases and surfaces a clear error
|
||||
message containing a truncated response snippet so future failures are
|
||||
diagnosable from the log alone.
|
||||
"""
|
||||
if not isinstance(response_text, str):
|
||||
raise json.JSONDecodeError("Empty memory update response (non-string content)", "", 0)
|
||||
|
||||
cleaned = _THINK_BLOCK_RE.sub("", response_text).strip()
|
||||
if not cleaned:
|
||||
raise json.JSONDecodeError("Empty memory update response (only reasoning content)", response_text, 0)
|
||||
|
||||
# 1. Try fenced code block first — favours the model's explicit framing.
|
||||
fenced_match = _FENCED_BLOCK_RE.search(cleaned)
|
||||
if fenced_match:
|
||||
candidate = fenced_match.group(1).strip()
|
||||
if candidate:
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
# Fall through to balanced-brace extraction; the fenced block
|
||||
# may have contained truncated JSON or interleaved prose.
|
||||
pass
|
||||
|
||||
# 2. Try balanced-brace JSON extraction anywhere in the text.
|
||||
balanced = _extract_first_json_object(cleaned)
|
||||
if balanced is not None:
|
||||
try:
|
||||
return json.loads(balanced)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# 3. Last resort: try parsing the whole cleaned text directly. This
|
||||
# covers responses that already are pure JSON.
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError as exc:
|
||||
snippet = response_text.strip()
|
||||
if len(snippet) > 400:
|
||||
snippet = snippet[:400] + "...[truncated]"
|
||||
raise json.JSONDecodeError(
|
||||
f"{exc.msg}; raw response head: {snippet!r}",
|
||||
exc.doc,
|
||||
exc.pos,
|
||||
) from exc
|
||||
|
||||
|
||||
class MemoryUpdater:
|
||||
"""Updates memory using LLM based on conversation context."""
|
||||
|
||||
def __init__(self, model_name: str | None = None):
|
||||
"""Initialize the memory updater.
|
||||
|
||||
Args:
|
||||
model_name: Optional model name to use. If None, uses config or default.
|
||||
"""
|
||||
self._model_name = model_name
|
||||
|
||||
def _get_model(self):
|
||||
"""Get the model for memory updates."""
|
||||
config = get_memory_config()
|
||||
model_name = self._model_name or config.model_name
|
||||
return create_chat_model(name=model_name, thinking_enabled=False)
|
||||
|
||||
def _build_correction_hint(
|
||||
self,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
) -> str:
|
||||
"""Build optional prompt hints for correction and reinforcement signals."""
|
||||
correction_hint = ""
|
||||
if correction_detected:
|
||||
correction_hint = (
|
||||
"IMPORTANT: Explicit correction signals were detected in this conversation. "
|
||||
"Pay special attention to what the agent got wrong, what the user corrected, "
|
||||
"and record the correct approach as a fact with category "
|
||||
'"correction" and confidence >= 0.95 when appropriate.'
|
||||
)
|
||||
if reinforcement_detected:
|
||||
reinforcement_hint = (
|
||||
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
|
||||
"The user explicitly confirmed the agent's approach was correct or helpful. "
|
||||
"Record the confirmed approach, style, or preference as a fact with category "
|
||||
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
|
||||
)
|
||||
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
|
||||
|
||||
return correction_hint
|
||||
|
||||
def _prepare_update_prompt(
|
||||
self,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
user_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], str] | None:
|
||||
"""Load memory and build the update prompt for a conversation."""
|
||||
config = get_memory_config()
|
||||
if not config.enabled or not messages:
|
||||
return None
|
||||
|
||||
current_memory = get_memory_data(agent_name, user_id=user_id)
|
||||
conversation_text = format_conversation_for_update(messages)
|
||||
if not conversation_text.strip():
|
||||
return None
|
||||
|
||||
correction_hint = self._build_correction_hint(
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
prompt = MEMORY_UPDATE_PROMPT.format(
|
||||
current_memory=json.dumps(current_memory, indent=2),
|
||||
conversation=conversation_text,
|
||||
correction_hint=correction_hint,
|
||||
)
|
||||
return current_memory, prompt
|
||||
|
||||
def _finalize_update(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
response_content: Any,
|
||||
thread_id: str | None,
|
||||
agent_name: str | None,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Parse the model response, apply updates, and persist memory."""
|
||||
response_text = _extract_text(response_content)
|
||||
update_data = _parse_memory_update_response(response_text)
|
||||
# Deep-copy before in-place mutation so a subsequent save() failure
|
||||
# cannot corrupt the still-cached original object reference.
|
||||
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
|
||||
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
|
||||
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
|
||||
|
||||
async def aupdate_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Update memory asynchronously by delegating to the sync path.
|
||||
|
||||
Uses ``asyncio.to_thread`` to run the *sync* ``model.invoke()`` path
|
||||
in a worker thread so no second event loop is created and the
|
||||
langchain async httpx client pool (shared with the lead agent) is
|
||||
never touched. This eliminates the cross-loop connection-reuse bug
|
||||
described in issue #2615.
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _do_update_memory_sync(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Pure-sync memory update using ``model.invoke()``.
|
||||
|
||||
Uses the *sync* LLM call path so no event loop is created. This
|
||||
guarantees that the langchain provider's globally cached async
|
||||
httpx ``AsyncClient`` / connection pool (the one shared with the
|
||||
lead agent) is never touched — no cross-loop connection reuse is
|
||||
possible.
|
||||
"""
|
||||
try:
|
||||
prepared = self._prepare_update_prompt(
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
if prepared is None:
|
||||
return False
|
||||
|
||||
current_memory, prompt = prepared
|
||||
model = self._get_model()
|
||||
response = model.invoke(prompt, config={"run_name": "memory_agent"})
|
||||
return self._finalize_update(
|
||||
current_memory=current_memory,
|
||||
response_content=response.content,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("Failed to parse LLM response for memory update: %s", e)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.exception("Memory update failed: %s", e)
|
||||
return False
|
||||
|
||||
def update_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Synchronously update memory using the sync LLM path.
|
||||
|
||||
Uses ``model.invoke()`` (sync HTTP) which operates on a completely
|
||||
separate connection pool from the async ``AsyncClient`` shared by
|
||||
the lead agent. This eliminates the cross-loop connection-reuse
|
||||
bug described in issue #2615.
|
||||
|
||||
When called from within a running event loop (e.g. from a LangGraph
|
||||
node), the blocking sync call is offloaded to a thread pool so the
|
||||
caller's loop is not blocked.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
thread_id: Optional thread ID for tracking source.
|
||||
agent_name: If provided, updates per-agent memory. If None, updates global memory.
|
||||
correction_detected: Whether recent turns include an explicit correction signal.
|
||||
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
|
||||
user_id: If provided, scopes memory to a specific user.
|
||||
|
||||
Returns:
|
||||
True if update was successful, False otherwise.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop is not None and loop.is_running():
|
||||
try:
|
||||
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
return future.result()
|
||||
except Exception:
|
||||
logger.exception("Failed to offload memory update to executor")
|
||||
return False
|
||||
|
||||
return self._do_update_memory_sync(
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _apply_updates(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
update_data: dict[str, Any],
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Apply LLM-generated updates to memory.
|
||||
|
||||
Args:
|
||||
current_memory: Current memory data.
|
||||
update_data: Updates from LLM.
|
||||
thread_id: Optional thread ID for tracking.
|
||||
|
||||
Returns:
|
||||
Updated memory data.
|
||||
"""
|
||||
config = get_memory_config()
|
||||
now = utc_now_iso_z()
|
||||
|
||||
# Update user sections
|
||||
user_updates = update_data.get("user", {})
|
||||
for section in ["workContext", "personalContext", "topOfMind"]:
|
||||
section_data = user_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory["user"][section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# Update history sections
|
||||
history_updates = update_data.get("history", {})
|
||||
for section in ["recentMonths", "earlierContext", "longTermBackground"]:
|
||||
section_data = history_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory["history"][section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# Remove facts
|
||||
facts_to_remove = set(update_data.get("factsToRemove", []))
|
||||
if facts_to_remove:
|
||||
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
|
||||
|
||||
# Add new facts
|
||||
existing_fact_keys = {fact_key for fact_key in (_fact_content_key(fact.get("content")) for fact in current_memory.get("facts", [])) if fact_key is not None}
|
||||
new_facts = update_data.get("newFacts", [])
|
||||
for fact in new_facts:
|
||||
confidence = fact.get("confidence", 0.5)
|
||||
if confidence >= config.fact_confidence_threshold:
|
||||
raw_content = fact.get("content", "")
|
||||
if not isinstance(raw_content, str):
|
||||
continue
|
||||
normalized_content = raw_content.strip()
|
||||
fact_key = _fact_content_key(normalized_content)
|
||||
if fact_key is not None and fact_key in existing_fact_keys:
|
||||
continue
|
||||
|
||||
fact_entry = {
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": fact.get("category", "context"),
|
||||
"confidence": confidence,
|
||||
"createdAt": now,
|
||||
"source": thread_id or "unknown",
|
||||
}
|
||||
source_error = fact.get("sourceError")
|
||||
if isinstance(source_error, str):
|
||||
normalized_source_error = source_error.strip()
|
||||
if normalized_source_error:
|
||||
fact_entry["sourceError"] = normalized_source_error
|
||||
current_memory["facts"].append(fact_entry)
|
||||
if fact_key is not None:
|
||||
existing_fact_keys.add(fact_key)
|
||||
|
||||
# Enforce max facts limit
|
||||
if len(current_memory["facts"]) > config.max_facts:
|
||||
# Sort by confidence and keep top ones
|
||||
current_memory["facts"] = sorted(
|
||||
current_memory["facts"],
|
||||
key=lambda f: f.get("confidence", 0),
|
||||
reverse=True,
|
||||
)[: config.max_facts]
|
||||
|
||||
return current_memory
|
||||
|
||||
|
||||
def update_memory_from_conversation(
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Convenience function to update memory from a conversation.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
thread_id: Optional thread ID.
|
||||
agent_name: If provided, updates per-agent memory. If None, updates global memory.
|
||||
correction_detected: Whether recent turns include an explicit correction signal.
|
||||
reinforcement_detected: Whether recent turns include a positive reinforcement signal.
|
||||
user_id: If provided, scopes memory to a specific user.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise.
|
||||
"""
|
||||
updater = MemoryUpdater()
|
||||
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
|
||||
+245
-110
@@ -1,110 +1,245 @@
|
||||
"""Middleware for memory mechanism."""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langgraph.config import get_config
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
|
||||
from deerflow.agents.memory.queue import get_memory_queue
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.memory_config import MemoryConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryMiddlewareState(AgentState):
|
||||
"""Compatible with the `ThreadState` schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
"""Middleware that queues conversation for memory update after agent execution.
|
||||
|
||||
This middleware:
|
||||
1. After each agent execution, queues the conversation for memory update
|
||||
2. Only includes user inputs and final assistant responses (ignores tool calls)
|
||||
3. The queue uses debouncing to batch multiple updates together
|
||||
4. Memory is updated asynchronously via LLM summarization
|
||||
"""
|
||||
|
||||
state_schema = MemoryMiddlewareState
|
||||
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
|
||||
"""Initialize the MemoryMiddleware.
|
||||
|
||||
Args:
|
||||
agent_name: If provided, memory is stored per-agent. If None, uses global memory.
|
||||
memory_config: Explicit memory config. When omitted, legacy global
|
||||
config fallback is used.
|
||||
"""
|
||||
super().__init__()
|
||||
self._agent_name = agent_name
|
||||
self._memory_config = memory_config
|
||||
|
||||
@override
|
||||
def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
"""Queue conversation for memory update after agent completes.
|
||||
|
||||
Args:
|
||||
state: The current agent state.
|
||||
runtime: The runtime context.
|
||||
|
||||
Returns:
|
||||
None (no state changes needed from this middleware).
|
||||
"""
|
||||
config = self._memory_config or get_memory_config()
|
||||
if not config.enabled:
|
||||
return None
|
||||
|
||||
# Get thread ID from runtime context first, then fall back to LangGraph's configurable metadata
|
||||
thread_id = runtime.context.get("thread_id") if runtime.context else None
|
||||
if thread_id is None:
|
||||
config_data = get_config()
|
||||
thread_id = config_data.get("configurable", {}).get("thread_id")
|
||||
if not thread_id:
|
||||
logger.debug("No thread_id in context, skipping memory update")
|
||||
return None
|
||||
|
||||
# Get messages from state
|
||||
messages = state.get("messages", [])
|
||||
if not messages:
|
||||
logger.debug("No messages in state, skipping memory update")
|
||||
return None
|
||||
|
||||
# Filter to only keep user inputs and final assistant responses
|
||||
filtered_messages = filter_messages_for_memory(messages)
|
||||
|
||||
# Only queue if there's meaningful conversation
|
||||
# At minimum need one user message and one assistant response
|
||||
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
|
||||
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
|
||||
|
||||
if not user_messages or not assistant_messages:
|
||||
return None
|
||||
|
||||
# Queue the filtered conversation for memory update
|
||||
correction_detected = detect_correction(filtered_messages)
|
||||
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
|
||||
# Capture user_id at enqueue time while the request context is still alive.
|
||||
# threading.Timer fires on a different thread where ContextVar values are not
|
||||
# propagated, so we must store user_id explicitly in ConversationContext.
|
||||
user_id = get_effective_user_id()
|
||||
queue = get_memory_queue()
|
||||
queue.add(
|
||||
thread_id=thread_id,
|
||||
messages=filtered_messages,
|
||||
agent_name=self._agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
|
||||
return None
|
||||
"""记忆机制中间件 (Memory V2)。
|
||||
|
||||
本中间件负责两件事:
|
||||
|
||||
1. **wrap_model_call** —— Hindsight 动态召回注入。
|
||||
每次 LLM 调用前,用最近一条用户消息作为 query 召回 Hindsight 记忆,
|
||||
作为一条带 ``<memory-context>`` 标签的消息**临时**追加到本次模型请求里。
|
||||
关键:用 ``wrap_model_call`` 而非 ``before_model`` —— 注入只作用于这一次
|
||||
模型调用,不写入持久 state,因此不会泄漏给前端、也不会在历史里堆积。
|
||||
仅在 ``provider="hindsight"`` 且配置允许时生效;builtin 的 frozen
|
||||
snapshot 走系统提示词,不在这里。
|
||||
|
||||
2. **after_agent** —— Hindsight 自动 retain。
|
||||
Hindsight ``auto_retain`` 开启时,把本轮对话写入工作 bank。
|
||||
|
||||
LLM 主动写入走 ``memory`` / ``hindsight_*`` 工具,不经过本中间件。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langgraph.config import get_config
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.agents.memory.manager import build_memory_manager
|
||||
from deerflow.agents.memory.message_processing import count_user_turns, filter_messages_for_memory
|
||||
from deerflow.config.memory_config import get_effective_memory_config, get_memory_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.memory_config import MemoryConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _coerce_text(content: Any) -> str:
|
||||
"""把消息 content (可能是 str 或多模态 block 列表) 压成纯文本。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text", "")))
|
||||
return " ".join(parts)
|
||||
return str(content or "")
|
||||
|
||||
|
||||
def _latest_human_text(messages: list) -> str:
|
||||
"""返回最近一条用户消息的纯文本。"""
|
||||
for msg in reversed(messages):
|
||||
if isinstance(msg, HumanMessage):
|
||||
return _coerce_text(msg.content).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _last_exchange(filtered_messages: list) -> tuple[str, str]:
|
||||
"""从过滤后的消息里取最后一轮 (用户输入, 助手回复) 的纯文本。"""
|
||||
user_text = ""
|
||||
assistant_text = ""
|
||||
for msg in reversed(filtered_messages):
|
||||
if not assistant_text and isinstance(msg, AIMessage):
|
||||
assistant_text = _coerce_text(msg.content).strip()
|
||||
elif not user_text and isinstance(msg, HumanMessage):
|
||||
user_text = _coerce_text(msg.content).strip()
|
||||
if user_text and assistant_text:
|
||||
break
|
||||
return user_text, assistant_text
|
||||
|
||||
|
||||
class MemoryMiddlewareState(AgentState):
|
||||
"""Compatible with the `ThreadState` schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
"""记忆中间件:wrap_model_call 注入 Hindsight 召回,after_agent 自动 retain。
|
||||
|
||||
Memory V2 说明:
|
||||
- builtin 本地记忆的 frozen snapshot 走系统提示词 (``_get_memory_context``)。
|
||||
- Hindsight 动态召回走本中间件 ``wrap_model_call`` —— 临时注入,不落 state。
|
||||
"""
|
||||
|
||||
state_schema = MemoryMiddlewareState
|
||||
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
|
||||
"""初始化 MemoryMiddleware。
|
||||
|
||||
Args:
|
||||
agent_name: 提供时记忆按 agent 隔离;None 用默认 agent。
|
||||
memory_config: 显式记忆配置;省略时用全局配置。
|
||||
"""
|
||||
super().__init__()
|
||||
self._agent_name = agent_name
|
||||
self._memory_config = memory_config
|
||||
|
||||
# ---- 运行时上下文解析 -------------------------------------------------
|
||||
|
||||
def _resolve_thread_id(self, runtime: Runtime | None) -> str | None:
|
||||
"""从 runtime context 取 thread_id,回退到 LangGraph configurable。"""
|
||||
thread_id = None
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
thread_id = runtime.context.get("thread_id")
|
||||
if thread_id:
|
||||
return thread_id
|
||||
try:
|
||||
return get_config().get("configurable", {}).get("thread_id")
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
def _resolve_agent_id(self, runtime: Runtime | None) -> str:
|
||||
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。"""
|
||||
if self._agent_name:
|
||||
return self._agent_name
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
agent_id = runtime.context.get("agent_id")
|
||||
if agent_id:
|
||||
return str(agent_id)
|
||||
return "default"
|
||||
|
||||
# ---- wrap_model_call:Hindsight 动态召回注入 -------------------------
|
||||
|
||||
def _build_recall_message(self, request: Any) -> HumanMessage | None:
|
||||
"""按本次模型请求构建一条 Hindsight 召回消息;无召回内容时返回 None。
|
||||
|
||||
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
|
||||
背景信息而非用户的新输入。
|
||||
"""
|
||||
config = self._memory_config or get_memory_config()
|
||||
if not config.enabled:
|
||||
return None
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
effective_config = get_effective_memory_config(user_id)
|
||||
if not effective_config.enabled or not effective_config.injection.enabled:
|
||||
return None
|
||||
# 只有 Hindsight 提供动态召回;builtin 走系统提示词。
|
||||
if effective_config.provider != "hindsight" or not effective_config.injection.include_hindsight:
|
||||
return None
|
||||
|
||||
messages = getattr(request, "messages", None) or []
|
||||
query = _latest_human_text(messages)
|
||||
if not query:
|
||||
return None
|
||||
|
||||
runtime = getattr(request, "runtime", None)
|
||||
thread_id = self._resolve_thread_id(runtime)
|
||||
try:
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=self._resolve_agent_id(runtime),
|
||||
thread_id=thread_id,
|
||||
config=effective_config,
|
||||
)
|
||||
recall = manager.prefetch_all(query, thread_id=thread_id or "")
|
||||
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
|
||||
logger.debug("Hindsight 召回注入失败 (非致命):%s", exc)
|
||||
return None
|
||||
|
||||
if not recall or not recall.strip():
|
||||
return None
|
||||
|
||||
tag = effective_config.injection.context_tag
|
||||
fenced = (
|
||||
f"<{tag}>\n"
|
||||
"[系统提示:以下是召回的记忆上下文,不是用户的新输入,作为背景信息参考。]\n\n"
|
||||
f"{recall}\n"
|
||||
f"</{tag}>"
|
||||
)
|
||||
return HumanMessage(content=fenced)
|
||||
|
||||
@override
|
||||
def wrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
"""模型调用前临时注入 Hindsight 召回 (同步路径)。"""
|
||||
recall_msg = self._build_recall_message(request)
|
||||
if recall_msg is not None:
|
||||
request = request.override(messages=[*request.messages, recall_msg])
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
"""模型调用前临时注入 Hindsight 召回 (异步路径)。
|
||||
|
||||
召回过程是同步阻塞 (含 Hindsight HTTP),放到线程池里跑以免阻塞事件循环。
|
||||
"""
|
||||
recall_msg = await asyncio.to_thread(self._build_recall_message, request)
|
||||
if recall_msg is not None:
|
||||
request = request.override(messages=[*request.messages, recall_msg])
|
||||
return await handler(request)
|
||||
|
||||
# ---- after_agent:Hindsight 自动 retain -------------------------------
|
||||
|
||||
@override
|
||||
def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
"""agent 完成后把本轮对话写入 Hindsight 工作 bank。
|
||||
|
||||
Returns:
|
||||
None (本中间件不改状态)。
|
||||
"""
|
||||
config = self._memory_config or get_memory_config()
|
||||
if not config.enabled:
|
||||
return None
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
effective_config = get_effective_memory_config(user_id)
|
||||
if effective_config.provider != "hindsight" or not effective_config.hindsight.auto_retain:
|
||||
return None
|
||||
|
||||
messages = state.get("messages", [])
|
||||
if not messages:
|
||||
return None
|
||||
filtered_messages = filter_messages_for_memory(messages)
|
||||
user_text, assistant_text = _last_exchange(filtered_messages)
|
||||
if not user_text or not assistant_text:
|
||||
return None
|
||||
|
||||
# 轮次序号按对话历史算出 —— provider 每轮被重建、无法自行计数,
|
||||
# retain_every_n_turns 的节流必须靠这个外部传入的值。
|
||||
turn_index = count_user_turns(filtered_messages)
|
||||
|
||||
thread_id = self._resolve_thread_id(runtime)
|
||||
try:
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=self._resolve_agent_id(runtime),
|
||||
thread_id=thread_id,
|
||||
config=effective_config,
|
||||
)
|
||||
manager.sync_all(
|
||||
user_text,
|
||||
assistant_text,
|
||||
thread_id=thread_id or "",
|
||||
turn_index=turn_index,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
|
||||
logger.warning("Hindsight retain 失败:%s", exc)
|
||||
|
||||
return None
|
||||
+8
@@ -131,6 +131,14 @@ def _build_runtime_middlewares(
|
||||
|
||||
middlewares.append(SandboxAuditMiddleware())
|
||||
middlewares.append(ToolErrorHandlingMiddleware())
|
||||
|
||||
# Tool / skill call metrics — appended last so it sits *deepest* in the
|
||||
# wrap_tool_call chain: its handler is the raw tool, so it observes the
|
||||
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
|
||||
# an error ToolMessage) and the raw return value.
|
||||
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
|
||||
|
||||
middlewares.append(ToolMetricsMiddleware())
|
||||
return middlewares
|
||||
|
||||
|
||||
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
"""Middleware that records per-tool-call metrics for the admin audit page.
|
||||
|
||||
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
|
||||
- Successes record the wall-clock duration.
|
||||
- Failures record a coarse ``error_category`` plus a one-line message.
|
||||
|
||||
Two failure shapes are captured:
|
||||
|
||||
1. **Raised exceptions** — the tool blew up. We record, then re-raise so the
|
||||
outer :class:`ToolErrorHandlingMiddleware` still converts it into an error
|
||||
``ToolMessage`` for the model.
|
||||
2. **Self-handled errors** — search/retrieval tools (Serper, Tavily, Jina, …)
|
||||
catch their own HTTP 429/403/timeout and *return* an error payload instead
|
||||
of raising. Those never reach an ``except`` block, so we also inspect the
|
||||
returned ``ToolMessage``: an ``status == "error"`` message, or content that
|
||||
looks like an error string, is bucketed too.
|
||||
|
||||
This middleware is appended **after** ``ToolErrorHandlingMiddleware`` so it
|
||||
sits deepest in the ``wrap_tool_call`` chain — its handler is the raw tool, so
|
||||
it observes the unconverted exception and the raw return value.
|
||||
|
||||
Writes go through a best-effort store: a DB outage logs a warning and the
|
||||
agent runtime keeps running.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.errors import GraphBubbleUp
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _trim(text: str, limit: int = 500) -> str:
|
||||
text = (text or "").strip()
|
||||
return text if len(text) <= limit else text[: limit - 3] + "..."
|
||||
|
||||
|
||||
def _classify_error(text: str) -> str:
|
||||
"""Bucket an error string into a coarse, admin-friendly category."""
|
||||
t = (text or "").lower()
|
||||
if any(k in t for k in ("429", "rate limit", "ratelimit", "too many request", "quota")):
|
||||
return "rate_limited"
|
||||
if any(k in t for k in ("403", "forbidden", "ip ban", "ip block", "blocked", "access denied", "banned")):
|
||||
return "ip_blocked"
|
||||
if any(k in t for k in ("401", "unauthorized", "api key", "apikey", "invalid key", "authentication", "credential")):
|
||||
return "auth_error"
|
||||
if any(k in t for k in ("timeout", "timed out", "deadline")):
|
||||
return "timeout"
|
||||
if any(k in t for k in ("connecterror", "connection error", "connection refused", "network", "dns", "unreachable", "ssl", "proxy")):
|
||||
return "network_error"
|
||||
if any(k in t for k in ("404", "not found")):
|
||||
return "not_found"
|
||||
if any(k in t for k in ("no results", "no result", "0 results", "empty result")):
|
||||
return "empty_result"
|
||||
return "tool_error"
|
||||
|
||||
|
||||
def _content_text(content: Any) -> str:
|
||||
"""Flatten a tool result's content (string or block list) into text."""
|
||||
if content is None:
|
||||
return ""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
text = block.get("text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
return " ".join(parts)
|
||||
return str(content)
|
||||
|
||||
|
||||
def _looks_like_error(text: str) -> bool:
|
||||
"""Heuristically detect a self-handled error payload returned as content.
|
||||
|
||||
Deliberately conservative — only an explicit ``Error:`` prefix or a JSON
|
||||
object whose first key is ``error`` counts, so a normal result that merely
|
||||
mentions the word "error" is not misclassified.
|
||||
"""
|
||||
head = (text or "").lstrip()[:64].lower()
|
||||
return head.startswith("error:") or head.startswith('{"error"') or head.startswith("{'error'")
|
||||
|
||||
|
||||
# Skill tools whose ``name`` argument names one specific Agent Skill directly.
|
||||
_SKILL_NAME_ARG_TOOLS = {"skill_view", "skill_manage", "skill_archive"}
|
||||
|
||||
# Matches a skill directory embedded in any tool-argument path. Skills live
|
||||
# under ``skills/{public,custom}/{name}/`` — both inside the sandbox
|
||||
# (``/mnt/skills/...``) and on the host. The lead agent's system prompt tells
|
||||
# it to load a skill by calling ``read_file`` on that SKILL.md path (it does
|
||||
# NOT call a dedicated "use skill" tool), so the skill name has to be recovered
|
||||
# from the path argument of whatever tool touched it (read_file / bash / ls /
|
||||
# str_replace / write_file ...).
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
|
||||
|
||||
def _skill_name_from_call(tool_name: str | None, tool_call: Any) -> str | None:
|
||||
"""Recover the Agent Skill a tool call targets, or ``None``.
|
||||
|
||||
Two detection paths:
|
||||
|
||||
1. ``skill_view`` / ``skill_manage`` / ``skill_archive`` take an explicit
|
||||
``name`` argument.
|
||||
2. Every other tool references a skill by *path*. The canonical skill
|
||||
invocation is ``read_file('/mnt/skills/<cat>/<name>/SKILL.md')``; helper
|
||||
files and ``bash`` script runs hit the same directory. So we scan the
|
||||
string arguments for the ``skills/{public,custom}/{name}/`` shape.
|
||||
"""
|
||||
if not isinstance(tool_call, dict):
|
||||
return None
|
||||
args = tool_call.get("args")
|
||||
if not isinstance(args, dict):
|
||||
return None
|
||||
|
||||
# Case 1 — explicit ``name`` argument on the skill tools.
|
||||
if tool_name in _SKILL_NAME_ARG_TOOLS:
|
||||
raw = args.get("name")
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
return raw.strip()
|
||||
|
||||
# Case 2 — a skill path inside any string argument.
|
||||
for value in args.values():
|
||||
if isinstance(value, str):
|
||||
match = _SKILL_PATH_RE.search(value)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def _evaluate_skill_view(text: str) -> tuple[str, str | None, str | None]:
|
||||
"""Classify a ``skill_view`` return.
|
||||
|
||||
``skill_view`` never raises — on failure it returns a plain string
|
||||
(``Skill 'x' not found.`` / ``Error reading skill 'x': ...``); on success
|
||||
it returns the SKILL.md content. So success/failure must be read off the
|
||||
returned text, not an exception or ``status`` flag.
|
||||
"""
|
||||
head = (text or "").lstrip()
|
||||
low = head[:200].lower()
|
||||
if head.startswith("Skill '") and "not found" in low:
|
||||
return "error", "not_found", _trim(text)
|
||||
if low.startswith("error reading skill"):
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
return "success", None, None
|
||||
|
||||
|
||||
def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | None, str | None]:
|
||||
"""Return ``(status, error_category, error_message)`` for a tool return."""
|
||||
if isinstance(result, ToolMessage):
|
||||
text = _content_text(result.content)
|
||||
if getattr(result, "status", None) == "error":
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
# skill_view swallows its own failures into a plain string return.
|
||||
if tool_name == "skill_view":
|
||||
return _evaluate_skill_view(text)
|
||||
if _looks_like_error(text):
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
return "success", None, None
|
||||
# Command (control flow) or any other shape — treat as a successful call.
|
||||
return "success", None, None
|
||||
|
||||
|
||||
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Wrap each tool call to time it and persist a metric row."""
|
||||
|
||||
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._store_override = store
|
||||
|
||||
# -- store / context ----------------------------------------------------
|
||||
|
||||
def _store(self) -> ToolMetricsStore | None:
|
||||
if self._store_override is not None:
|
||||
return self._store_override
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
except Exception:
|
||||
return None
|
||||
return make_tool_metrics_store(sf)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in keys:
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
config = getattr(runtime, "config", None)
|
||||
if isinstance(config, dict):
|
||||
configurable = config.get("configurable") if isinstance(config.get("configurable"), dict) else {}
|
||||
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
|
||||
for bucket in (configurable, metadata):
|
||||
for key in keys:
|
||||
value = bucket.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
def _build_base_record(self, request: ToolCallRequest) -> ToolMetricRecord:
|
||||
tool_call = getattr(request, "tool_call", None) or {}
|
||||
tool_name = tool_call.get("name") if isinstance(tool_call, dict) else None
|
||||
tool_name = str(tool_name) if tool_name else None
|
||||
return ToolMetricRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=_now(),
|
||||
user_id=get_effective_user_id(),
|
||||
thread_id=self._runtime_lookup(request, ("thread_id",)),
|
||||
run_id=self._runtime_lookup(request, ("run_id", "langgraph_run_id")),
|
||||
agent_name=self._runtime_lookup(request, ("agent_name", "agent_id")),
|
||||
skill_name=_skill_name_from_call(tool_name, tool_call),
|
||||
tool_name=tool_name,
|
||||
)
|
||||
|
||||
# -- finalize -----------------------------------------------------------
|
||||
|
||||
def _finalize_result(self, record: ToolMetricRecord, result: Any, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status, record.error_category, record.error_message = _evaluate_result(
|
||||
result, record.tool_name
|
||||
)
|
||||
|
||||
def _finalize_error(self, record: ToolMetricRecord, exc: BaseException, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status = "error"
|
||||
name = exc.__class__.__name__
|
||||
combined = f"{name}: {(str(exc) or name).strip()}"
|
||||
record.error_category = _classify_error(combined)
|
||||
record.error_message = _trim(combined)
|
||||
|
||||
def _flush(self, record: ToolMetricRecord) -> None:
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the tool path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
# -- wrap ---------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def wrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = handler(request)
|
||||
except GraphBubbleUp:
|
||||
# LangGraph control flow — not a tool failure.
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_result(record, result, started)
|
||||
self._flush(record)
|
||||
return result
|
||||
|
||||
@override
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = await handler(request)
|
||||
except GraphBubbleUp:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_result(record, result, started)
|
||||
self._flush(record)
|
||||
return result
|
||||
@@ -765,27 +765,22 @@ class DeerFlowClient:
|
||||
]
|
||||
}
|
||||
|
||||
def get_memory(self) -> dict:
|
||||
"""Get current memory data.
|
||||
def get_memory(self, agent: str | None = None) -> dict:
|
||||
"""读取 Memory V2 本地记忆 (USER.md / MEMORY.md)。
|
||||
|
||||
Args:
|
||||
agent: 可选 agent 名;省略表示默认 agent。
|
||||
|
||||
Returns:
|
||||
Memory data dict (see src/agents/memory/updater.py for structure).
|
||||
``{user_entries, memory_entries, user_usage, memory_usage}``。
|
||||
"""
|
||||
from deerflow.agents.memory.updater import get_memory_data
|
||||
|
||||
return get_memory_data(user_id=get_effective_user_id())
|
||||
|
||||
def export_memory(self) -> dict:
|
||||
"""Export current memory data for backup or transfer."""
|
||||
from deerflow.agents.memory.updater import get_memory_data
|
||||
|
||||
return get_memory_data(user_id=get_effective_user_id())
|
||||
|
||||
def import_memory(self, memory_data: dict) -> dict:
|
||||
"""Import and persist full memory data."""
|
||||
from deerflow.agents.memory.updater import import_memory_data
|
||||
|
||||
return import_memory_data(memory_data, user_id=get_effective_user_id())
|
||||
provider = self._build_memory_provider(agent)
|
||||
return {
|
||||
"user_entries": provider.read("user"),
|
||||
"memory_entries": provider.read("memory"),
|
||||
"user_usage": provider.usage("user"),
|
||||
"memory_usage": provider.usage("memory"),
|
||||
}
|
||||
|
||||
def get_model(self, name: str) -> dict | None:
|
||||
"""Get a specific model's configuration by name.
|
||||
@@ -949,83 +944,73 @@ class DeerFlowClient:
|
||||
return get_or_new_skill_storage().install_skill_from_archive(skill_path)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API — memory management
|
||||
# Public API — memory management (Memory V2)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def reload_memory(self) -> dict:
|
||||
"""Reload memory data from file, forcing cache invalidation.
|
||||
def _build_memory_provider(self, agent: str | None = None):
|
||||
"""按生效配置构建并初始化一个 V2 builtin provider。"""
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
Returns:
|
||||
The reloaded memory data dict.
|
||||
"""
|
||||
from deerflow.agents.memory.updater import reload_memory_data
|
||||
|
||||
return reload_memory_data(user_id=get_effective_user_id())
|
||||
|
||||
def clear_memory(self) -> dict:
|
||||
"""Clear all persisted memory data."""
|
||||
from deerflow.agents.memory.updater import clear_memory_data
|
||||
|
||||
return clear_memory_data(user_id=get_effective_user_id())
|
||||
|
||||
def create_memory_fact(self, content: str, category: str = "context", confidence: float = 0.5) -> dict:
|
||||
"""Create a single fact manually."""
|
||||
from deerflow.agents.memory.updater import create_memory_fact
|
||||
|
||||
return create_memory_fact(content=content, category=category, confidence=confidence)
|
||||
|
||||
def delete_memory_fact(self, fact_id: str) -> dict:
|
||||
"""Delete a single fact from memory by fact id."""
|
||||
from deerflow.agents.memory.updater import delete_memory_fact
|
||||
|
||||
return delete_memory_fact(fact_id)
|
||||
|
||||
def update_memory_fact(
|
||||
self,
|
||||
fact_id: str,
|
||||
content: str | None = None,
|
||||
category: str | None = None,
|
||||
confidence: float | None = None,
|
||||
) -> dict:
|
||||
"""Update a single fact manually, preserving omitted fields."""
|
||||
from deerflow.agents.memory.updater import update_memory_fact
|
||||
|
||||
return update_memory_fact(
|
||||
fact_id=fact_id,
|
||||
content=content,
|
||||
category=category,
|
||||
confidence=confidence,
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
provider = BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
)
|
||||
provider.initialize(
|
||||
user_id=user_id,
|
||||
agent_id=agent or "default",
|
||||
base_dir=str(get_paths().base_dir),
|
||||
)
|
||||
return provider
|
||||
|
||||
def add_memory_entry(self, target: str, content: str, agent: str | None = None) -> dict:
|
||||
"""新增一条记忆条目。``target`` 为 'user'(用户画像) 或 'memory'(工作笔记)。"""
|
||||
return self._build_memory_provider(agent).add(target, content)
|
||||
|
||||
def replace_memory_entry(self, target: str, old_text: str, content: str, agent: str | None = None) -> dict:
|
||||
"""用 ``old_text`` 短串定位并替换一条记忆条目。"""
|
||||
return self._build_memory_provider(agent).replace(target, old_text, content)
|
||||
|
||||
def remove_memory_entry(self, target: str, old_text: str, agent: str | None = None) -> dict:
|
||||
"""用 ``old_text`` 短串定位并删除一条记忆条目。"""
|
||||
return self._build_memory_provider(agent).remove(target, old_text)
|
||||
|
||||
def get_memory_config(self) -> dict:
|
||||
"""Get memory system configuration.
|
||||
"""读取当前用户的生效记忆配置 (含 per-user 覆盖)。"""
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
|
||||
return get_effective_memory_config(get_effective_user_id()).model_dump()
|
||||
|
||||
def search_memory(self, query: str, mode: str = "recall", agent: str | None = None) -> dict:
|
||||
"""代理 Hindsight 检索。
|
||||
|
||||
Args:
|
||||
query: 检索查询。
|
||||
mode: ``recall`` 返回原始片段;``reflect`` 返回 LLM 综合结论。
|
||||
agent: 可选 agent 名。
|
||||
|
||||
Returns:
|
||||
Memory config dict.
|
||||
``{available, results, text}``;Hindsight 不可用时 available=False。
|
||||
"""
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
|
||||
config = get_memory_config()
|
||||
return {
|
||||
"enabled": config.enabled,
|
||||
"storage_path": config.storage_path,
|
||||
"debounce_seconds": config.debounce_seconds,
|
||||
"max_facts": config.max_facts,
|
||||
"fact_confidence_threshold": config.fact_confidence_threshold,
|
||||
"injection_enabled": config.injection_enabled,
|
||||
"max_injection_tokens": config.max_injection_tokens,
|
||||
}
|
||||
|
||||
def get_memory_status(self) -> dict:
|
||||
"""Get memory status: config + current data.
|
||||
|
||||
Returns:
|
||||
Dict with "config" and "data" keys.
|
||||
"""
|
||||
return {
|
||||
"config": self.get_memory_config(),
|
||||
"data": self.get_memory(),
|
||||
}
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
if not config.enabled or config.provider != "hindsight":
|
||||
return {"available": False, "results": [], "text": ""}
|
||||
provider = HindsightProvider(config.hindsight)
|
||||
if not provider.is_available():
|
||||
return {"available": False, "results": [], "text": ""}
|
||||
provider.initialize(user_id=user_id, agent_id=agent or "default", thread_id="", base_dir="")
|
||||
if mode == "reflect":
|
||||
return {"available": True, "results": [], "text": provider.reflect(query)}
|
||||
return {"available": True, "results": provider.recall(query), "text": ""}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API — file uploads
|
||||
|
||||
+37
-15
@@ -43,9 +43,27 @@ logger = logging.getLogger(__name__)
|
||||
DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest"
|
||||
DEFAULT_PORT = 8080
|
||||
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
PACKAGE_MANAGER_ENV_KEYS = (
|
||||
"PIP_INDEX_URL",
|
||||
"PIP_EXTRA_INDEX_URL",
|
||||
"PIP_TRUSTED_HOST",
|
||||
"PIP_CERT",
|
||||
"PIP_CLIENT_CERT",
|
||||
"UV_INDEX_URL",
|
||||
"UV_EXTRA_INDEX_URL",
|
||||
"UV_DEFAULT_INDEX",
|
||||
"UV_INDEX",
|
||||
"UV_TRUSTED_HOST",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"NO_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"no_proxy",
|
||||
)
|
||||
|
||||
|
||||
def _lock_file_exclusive(lock_file) -> None:
|
||||
@@ -142,9 +160,9 @@ class AioSandboxProvider(SandboxProvider):
|
||||
Local provider manages container lifecycle directly (start/stop).
|
||||
"""
|
||||
provisioner_url = self._config.get("provisioner_url")
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url)
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url, environment=self._config["environment"])
|
||||
|
||||
logger.info("Using local container sandbox backend")
|
||||
return LocalContainerBackend(
|
||||
@@ -172,20 +190,24 @@ class AioSandboxProvider(SandboxProvider):
|
||||
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
|
||||
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
|
||||
"mounts": sandbox_config.mounts or [],
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
|
||||
"provisioner_url": getattr(sandbox_config, "provisioner_url", None) or "",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key in PACKAGE_MANAGER_ENV_KEYS:
|
||||
value = os.environ.get(key)
|
||||
if value:
|
||||
resolved[key] = value
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
resolved[key] = str(value)
|
||||
return resolved
|
||||
|
||||
|
||||
+16
-12
@@ -40,14 +40,17 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
provisioner_url: http://provisioner:8002
|
||||
"""
|
||||
|
||||
def __init__(self, provisioner_url: str):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
def __init__(self, provisioner_url: str, environment: dict[str, str] | None = None):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
environment: Environment variables for sandbox pods. The remote
|
||||
provisioner may ignore this field if it does not support it.
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
self._environment = environment or {}
|
||||
|
||||
@property
|
||||
def provisioner_url(self) -> str:
|
||||
@@ -91,10 +94,11 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._provisioner_url}/api/sandboxes",
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
"environment": self._environment,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
+47
-26
@@ -11,8 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
|
||||
that contend for the lock wait via the default 5-second sqlite3
|
||||
busy timeout rather than failing immediately.
|
||||
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
|
||||
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
|
||||
No database is initialized.
|
||||
@@ -21,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
|
||||
to reference environment variables from .env:
|
||||
|
||||
database:
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -37,31 +42,40 @@ from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
|
||||
)
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
)
|
||||
sqlite_dir: str = Field(
|
||||
default=".deer-flow/data",
|
||||
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
||||
)
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
"Use $DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: postgresql://user:pass@host:5432/deerflow "
|
||||
"(the +asyncpg driver suffix is added automatically where needed)."
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"MySQL connection URL for the application ORM. "
|
||||
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
|
||||
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
|
||||
),
|
||||
)
|
||||
echo_sql: bool = Field(
|
||||
default=False,
|
||||
description="Echo all SQL statements to log (debug only).",
|
||||
)
|
||||
pool_size: int = Field(
|
||||
default=5,
|
||||
description="Connection pool size for the app ORM engine (postgres only).",
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
)
|
||||
|
||||
# -- Derived helpers (not user-configured) --
|
||||
@@ -94,9 +108,16 @@ class DatabaseConfig(BaseModel):
|
||||
"""SQLAlchemy async URL for the application ORM engine."""
|
||||
if self.backend == "sqlite":
|
||||
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
if url.startswith("mysql://"):
|
||||
url = url.replace("mysql://", "mysql+asyncmy://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
+315
-83
@@ -1,83 +1,315 @@
|
||||
"""Configuration for memory mechanism."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class MemoryConfig(BaseModel):
|
||||
"""Configuration for global memory mechanism."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description="Whether to enable memory mechanism",
|
||||
)
|
||||
storage_path: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Path to store memory data. "
|
||||
"If empty, defaults to per-user memory at `{base_dir}/users/{user_id}/memory.json`. "
|
||||
"Absolute paths are used as-is and opt out of per-user isolation "
|
||||
"(all users share the same file). "
|
||||
"Relative paths are resolved against `Paths.base_dir` "
|
||||
"(not the backend working directory). "
|
||||
"Note: if you previously set this to `.deer-flow/memory.json`, "
|
||||
"the file will now be resolved as `{base_dir}/.deer-flow/memory.json`; "
|
||||
"migrate existing data or use an absolute path to preserve the old location."
|
||||
),
|
||||
)
|
||||
storage_class: str = Field(
|
||||
default="deerflow.agents.memory.storage.FileMemoryStorage",
|
||||
description="The class path for memory storage provider",
|
||||
)
|
||||
debounce_seconds: int = Field(
|
||||
default=30,
|
||||
ge=1,
|
||||
le=300,
|
||||
description="Seconds to wait before processing queued updates (debounce)",
|
||||
)
|
||||
model_name: str | None = Field(
|
||||
default=None,
|
||||
description="Model name to use for memory updates (None = use default model)",
|
||||
)
|
||||
max_facts: int = Field(
|
||||
default=100,
|
||||
ge=10,
|
||||
le=500,
|
||||
description="Maximum number of facts to store",
|
||||
)
|
||||
fact_confidence_threshold: float = Field(
|
||||
default=0.7,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Minimum confidence threshold for storing facts",
|
||||
)
|
||||
injection_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Whether to inject memory into system prompt",
|
||||
)
|
||||
max_injection_tokens: int = Field(
|
||||
default=2000,
|
||||
ge=100,
|
||||
le=8000,
|
||||
description="Maximum tokens to use for memory injection",
|
||||
)
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
_memory_config: MemoryConfig = MemoryConfig()
|
||||
|
||||
|
||||
def get_memory_config() -> MemoryConfig:
|
||||
"""Get the current memory configuration."""
|
||||
return _memory_config
|
||||
|
||||
|
||||
def set_memory_config(config: MemoryConfig) -> None:
|
||||
"""Set the memory configuration."""
|
||||
global _memory_config
|
||||
_memory_config = config
|
||||
|
||||
|
||||
def load_memory_config_from_dict(config_dict: dict) -> None:
|
||||
"""Load memory configuration from a dictionary."""
|
||||
global _memory_config
|
||||
_memory_config = MemoryConfig(**config_dict)
|
||||
"""Memory V2 - 记忆子系统配置。
|
||||
|
||||
配置结构 (详见 docs/MEMORY_V2_DESIGN_ZH.md §9)
|
||||
==============================================
|
||||
``MemoryConfig`` 采用嵌套结构::
|
||||
|
||||
memory:
|
||||
enabled: true
|
||||
provider: hindsight # 外部 provider 选哪个 (builtin 始终启用)
|
||||
builtin: {...} # 本地 USER.md / MEMORY.md
|
||||
hindsight: {...} # Hindsight HTTP 客户端
|
||||
injection: {...} # 系统提示词注入
|
||||
security: {...} # 内容安全扫描
|
||||
|
||||
三层配置覆盖
|
||||
============
|
||||
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
|
||||
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
|
||||
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
|
||||
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
|
||||
4. Pydantic 默认值
|
||||
|
||||
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 嵌套子配置
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class BuiltinMemoryConfig(BaseModel):
|
||||
"""本地文件 provider (USER.md / MEMORY.md) 配置。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="是否启用 builtin 本地记忆 provider")
|
||||
memory_char_limit: int = Field(
|
||||
default=2200, ge=200, le=20000, description="MEMORY.md 字符配额"
|
||||
)
|
||||
user_char_limit: int = Field(
|
||||
default=1375, ge=200, le=20000, description="USER.md 字符配额"
|
||||
)
|
||||
deduplicate_on_load: bool = Field(
|
||||
default=True, description="加载时是否对条目去重"
|
||||
)
|
||||
|
||||
|
||||
class HindsightMemoryConfig(BaseModel):
|
||||
"""Hindsight 外部 provider 配置。"""
|
||||
|
||||
mode: str = Field(
|
||||
default="local_external",
|
||||
description="连接模式:cloud / local_embedded / local_external",
|
||||
)
|
||||
api_url: str = Field(default="http://localhost:8765", description="Hindsight API 地址")
|
||||
api_key: str = Field(default="", description="API key,建议用 $HINDSIGHT_API_KEY 环境变量")
|
||||
bank_id_template: str = Field(
|
||||
default="deerflow-user-{user_id}",
|
||||
description="全局 user bank 命名模板,占位符 {user_id}/{agent_id}/{thread_id}",
|
||||
)
|
||||
work_bank_id_template: str = Field(
|
||||
default="deerflow-work-{user_id}-{agent_id}",
|
||||
description="agent 私有工作 bank 命名模板",
|
||||
)
|
||||
memory_mode: str = Field(
|
||||
default="context",
|
||||
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
|
||||
)
|
||||
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
|
||||
bank_retain_mission: str = Field(default="", description="引导抽取内容的 mission")
|
||||
recall_budget: str = Field(default="mid", description="召回深度:low / mid / high")
|
||||
recall_max_tokens: int = Field(default=4096, ge=256, le=32768, description="召回结果 token 上限")
|
||||
recall_max_input_chars: int = Field(default=800, ge=64, le=8000, description="召回 query 长度上限")
|
||||
auto_recall: bool = Field(default=True, description="每轮对话前是否自动召回")
|
||||
auto_retain: bool = Field(default=True, description="是否自动写入对话")
|
||||
retain_async: bool = Field(default=True, description="retain 是否在 Hindsight 服务端异步处理")
|
||||
retain_every_n_turns: int = Field(default=1, ge=1, le=50, description="每 N 轮 retain 一次")
|
||||
retain_user_prefix: str = Field(default="User", description="自动 retain 时用户发言前缀")
|
||||
retain_assistant_prefix: str = Field(default="Assistant", description="自动 retain 时助手发言前缀")
|
||||
|
||||
|
||||
class InjectionConfig(BaseModel):
|
||||
"""系统提示词记忆注入配置。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="是否往系统提示词注入记忆")
|
||||
max_tokens: int = Field(default=2000, ge=100, le=8000, description="注入 token 上限")
|
||||
include_builtin: bool = Field(default=True, description="是否注入 builtin 本地记忆")
|
||||
include_hindsight: bool = Field(default=True, description="是否注入 Hindsight 召回结果")
|
||||
context_tag: str = Field(
|
||||
default="memory-context", description="包裹外部召回内容的标签名"
|
||||
)
|
||||
|
||||
|
||||
class SecurityConfig(BaseModel):
|
||||
"""记忆内容安全配置。"""
|
||||
|
||||
scan_content: bool = Field(default=True, description="是否扫描注入/外渗模式")
|
||||
block_invisible_unicode: bool = Field(default=True, description="是否阻断不可见 unicode")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 顶层配置
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class MemoryConfig(BaseModel):
|
||||
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="记忆子系统总开关")
|
||||
provider: str = Field(
|
||||
default="builtin",
|
||||
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
|
||||
)
|
||||
|
||||
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
|
||||
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
|
||||
injection: InjectionConfig = Field(default_factory=InjectionConfig)
|
||||
security: SecurityConfig = Field(default_factory=SecurityConfig)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# per-user 运行时覆盖
|
||||
# ===========================================================================
|
||||
|
||||
# 用户可在运行时覆盖的字段白名单 (点号路径)。
|
||||
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
|
||||
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"injection.enabled",
|
||||
"injection.max_tokens",
|
||||
"injection.include_builtin",
|
||||
"injection.include_hindsight",
|
||||
"hindsight.memory_mode",
|
||||
"hindsight.recall_budget",
|
||||
"hindsight.bank_mission",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
|
||||
|
||||
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
|
||||
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
def _set_nested(path: str, value: Any) -> None:
|
||||
if path not in USER_WRITABLE_FIELDS:
|
||||
logger.warning("忽略非白名单的记忆配置覆盖字段:%s", path)
|
||||
return
|
||||
section, field = path.split(".", 1)
|
||||
result.setdefault(section, {})[field] = value
|
||||
|
||||
for key, value in overrides.items():
|
||||
if isinstance(value, dict):
|
||||
for sub_key, sub_value in value.items():
|
||||
_set_nested(f"{key}.{sub_key}", sub_value)
|
||||
else:
|
||||
_set_nested(key, value)
|
||||
return result
|
||||
|
||||
|
||||
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
||||
"""递归合并两个 dict,``override`` 优先。返回新 dict,不修改入参。"""
|
||||
merged = copy.deepcopy(base)
|
||||
for key, value in override.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
def _load_json_file(path: Any) -> dict[str, Any]:
|
||||
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return {}
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 全局单例 + 加载入口
|
||||
# ===========================================================================
|
||||
|
||||
_memory_config: MemoryConfig = MemoryConfig()
|
||||
|
||||
|
||||
def get_memory_config() -> MemoryConfig:
|
||||
"""返回 config.yaml 级别的记忆配置 (不含 per-user 覆盖)。"""
|
||||
return _memory_config
|
||||
|
||||
|
||||
def set_memory_config(config: MemoryConfig) -> None:
|
||||
"""设置 config.yaml 级别的记忆配置 (测试与运行时重载用)。"""
|
||||
global _memory_config
|
||||
_memory_config = config
|
||||
|
||||
|
||||
def load_memory_config_from_dict(config_dict: dict) -> None:
|
||||
"""从 dict 加载记忆配置。"""
|
||||
global _memory_config
|
||||
_memory_config = MemoryConfig(**config_dict)
|
||||
|
||||
|
||||
def get_effective_memory_config(user_id: str) -> MemoryConfig:
|
||||
"""按三层覆盖优先级,返回某用户的生效记忆配置。
|
||||
|
||||
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
|
||||
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
|
||||
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
|
||||
"""
|
||||
base = _memory_config.model_dump()
|
||||
|
||||
try:
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
paths = get_paths()
|
||||
global_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.memory_global_overrides_file)
|
||||
)
|
||||
user_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.user_memory_config_file(user_id))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
|
||||
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
|
||||
try:
|
||||
return MemoryConfig(**merged)
|
||||
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
|
||||
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
|
||||
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
|
||||
|
||||
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
current = _load_json_file(path)
|
||||
filtered = _filter_to_whitelist(patch)
|
||||
merged = _deep_merge(current, filtered)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return merged
|
||||
|
||||
|
||||
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
|
||||
"""撤销某用户的 per-user 覆盖。
|
||||
|
||||
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
|
||||
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
if field is None:
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return {}
|
||||
|
||||
current = _load_json_file(path)
|
||||
if "." in field:
|
||||
section, sub_field = field.split(".", 1)
|
||||
if isinstance(current.get(section), dict):
|
||||
current[section].pop(sub_field, None)
|
||||
if not current[section]:
|
||||
current.pop(section, None)
|
||||
else:
|
||||
current.pop(field, None)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return current
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BuiltinMemoryConfig",
|
||||
"HindsightMemoryConfig",
|
||||
"InjectionConfig",
|
||||
"SecurityConfig",
|
||||
"MemoryConfig",
|
||||
"USER_WRITABLE_FIELDS",
|
||||
"get_memory_config",
|
||||
"set_memory_config",
|
||||
"load_memory_config_from_dict",
|
||||
"get_effective_memory_config",
|
||||
"save_user_memory_override",
|
||||
"delete_user_memory_override",
|
||||
]
|
||||
@@ -155,6 +155,33 @@ class Paths:
|
||||
"""Per-user per-agent memory: `{base_dir}/users/{user_id}/agents/{id}/memory.json`."""
|
||||
return self.user_dir(user_id) / "agents" / agent_name.lower() / "memory.json"
|
||||
|
||||
# --- Memory V2 布局 (USER.md / MEMORY.md / per-user 配置覆盖) ---
|
||||
# 详见 docs/MEMORY_V2_DESIGN_ZH.md §5.1。注意与 ``user_md_file`` 区分:
|
||||
# 后者是 base_dir 下的全局 USER.md (旧布局),这里是 per-user 的 V2 布局。
|
||||
|
||||
def user_profile_md_file(self, user_id: str) -> Path:
|
||||
"""Memory V2 用户画像文件:`{base_dir}/users/{user_id}/USER.md`。
|
||||
|
||||
存放 LLM 主动写入的 ``target="user"`` 记忆 (用户身份、偏好,跨 agent 共享)。
|
||||
"""
|
||||
return self.user_dir(user_id) / "USER.md"
|
||||
|
||||
def user_agent_memory_md_file(self, user_id: str, agent_name: str) -> Path:
|
||||
"""Memory V2 agent 工作笔记文件:`{base_dir}/users/{user_id}/agents/{id}/MEMORY.md`。
|
||||
|
||||
存放 LLM 主动写入的 ``target="memory"`` 记忆 (agent 私有工作笔记)。
|
||||
"""
|
||||
return self.user_dir(user_id) / "agents" / agent_name.lower() / "MEMORY.md"
|
||||
|
||||
def user_memory_config_file(self, user_id: str) -> Path:
|
||||
"""Memory V2 per-user 运行时配置覆盖:`{base_dir}/users/{user_id}/memory_config.json`。"""
|
||||
return self.user_dir(user_id) / "memory_config.json"
|
||||
|
||||
@property
|
||||
def memory_global_overrides_file(self) -> Path:
|
||||
"""Memory V2 全局配置覆盖 (管理员配):`{base_dir}/memory_user_overrides.json`。"""
|
||||
return self.base_dir / "memory_user_overrides.json"
|
||||
|
||||
def thread_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for a thread's data.
|
||||
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class AgentRow(Base):
|
||||
@@ -16,9 +17,9 @@ class AgentRow(Base):
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
|
||||
|
||||
|
||||
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return AIWritingSessionRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["AIWritingSessionRow", "AIWritingSessionRepository", "make_ai_writing_session_store"]
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""ORM model for AI writing sessions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import PortableLongText
|
||||
|
||||
|
||||
class AIWritingSessionRow(Base):
|
||||
__tablename__ = "ai_writing_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
|
||||
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
|
||||
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
|
||||
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
|
||||
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
"""SQLAlchemy-backed AI writing session storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class AIWritingSessionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: AIWritingSessionRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
for k in ("completed_interventions", "review_result", "transcript"):
|
||||
val = d.get(k)
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
d[k] = json.loads(val)
|
||||
except Exception:
|
||||
d[k] = None
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
title: str,
|
||||
user_intent: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = AIWritingSessionRow(
|
||||
id=id,
|
||||
user_id=resolved_user_id,
|
||||
title=title,
|
||||
user_intent=user_intent,
|
||||
status="in_progress",
|
||||
draft_title=None,
|
||||
draft_markdown=None,
|
||||
completed_interventions=None,
|
||||
review_result=None,
|
||||
transcript=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
title=_UNSET,
|
||||
status=_UNSET,
|
||||
draft_title=_UNSET,
|
||||
draft_markdown=_UNSET,
|
||||
completed_interventions=_UNSET,
|
||||
review_result=_UNSET,
|
||||
transcript=_UNSET,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
if title is not _UNSET:
|
||||
row.title = title
|
||||
if status is not _UNSET:
|
||||
row.status = status
|
||||
if draft_title is not _UNSET:
|
||||
row.draft_title = draft_title
|
||||
if draft_markdown is not _UNSET:
|
||||
row.draft_markdown = draft_markdown
|
||||
if completed_interventions is not _UNSET:
|
||||
row.completed_interventions = (
|
||||
json.dumps(completed_interventions, ensure_ascii=False)
|
||||
if completed_interventions is not None
|
||||
else None
|
||||
)
|
||||
if review_result is not _UNSET:
|
||||
row.review_result = (
|
||||
json.dumps(review_result, ensure_ascii=False)
|
||||
if review_result is not None
|
||||
else None
|
||||
)
|
||||
if transcript is not _UNSET:
|
||||
row.transcript = (
|
||||
json.dumps(transcript, ensure_ascii=False)
|
||||
if transcript is not None
|
||||
else None
|
||||
)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(
|
||||
self,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
|
||||
stmt = select(AIWritingSessionRow)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_older_than(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
only_finished: bool = False,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
|
||||
|
||||
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
|
||||
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
|
||||
|
||||
Args:
|
||||
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
|
||||
only_finished: True 时只清 status in (done/error),False 不看状态
|
||||
limit: 单批最多返回多少条,用于分批处理避免大事务
|
||||
"""
|
||||
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
|
||||
if only_finished:
|
||||
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
async def delete_by_ids(self, session_ids: list[str]) -> int:
|
||||
"""按 id 列表批量删除。返回实际删除的行数。"""
|
||||
if not session_ids:
|
||||
return 0
|
||||
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
from deerflow.persistence.article_types.sql import ArticleTypeRepository
|
||||
|
||||
|
||||
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return ArticleTypeRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""ORM model for AI writing article types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class ArticleTypeRow(Base):
|
||||
__tablename__ = "ai_writing_article_types"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""SQLAlchemy-backed article type storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class ArticleTypeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: ArticleTypeRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
description: str | None = None,
|
||||
prompt_hint: str | None = None,
|
||||
default_word_count: int = 800,
|
||||
sort_order: int = 0,
|
||||
) -> dict:
|
||||
now = datetime.now(UTC)
|
||||
row = ArticleTypeRow(
|
||||
id=str(uuid.uuid4()),
|
||||
key=key,
|
||||
label=label,
|
||||
description=description,
|
||||
prompt_hint=prompt_hint,
|
||||
default_word_count=default_word_count,
|
||||
sort_order=sort_order,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
article_type_id: str,
|
||||
*,
|
||||
key=_UNSET,
|
||||
label=_UNSET,
|
||||
description=_UNSET,
|
||||
prompt_hint=_UNSET,
|
||||
default_word_count=_UNSET,
|
||||
sort_order=_UNSET,
|
||||
) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
if key is not _UNSET:
|
||||
row.key = key
|
||||
if label is not _UNSET:
|
||||
row.label = label
|
||||
if description is not _UNSET:
|
||||
row.description = description
|
||||
if prompt_hint is not _UNSET:
|
||||
row.prompt_hint = prompt_hint
|
||||
if default_word_count is not _UNSET:
|
||||
row.default_word_count = default_word_count
|
||||
if sort_order is not _UNSET:
|
||||
row.sort_order = sort_order
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(self, article_type_id: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_key(self, key: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(self) -> list[dict]:
|
||||
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(self, article_type_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_keys(self) -> list[str]:
|
||||
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars())
|
||||
@@ -10,10 +10,13 @@ None and fall back to in-memory implementations.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
@@ -23,11 +26,37 @@ def _json_serializer(obj: object) -> str:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
@@ -65,8 +94,8 @@ async def init_engine(
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", or "postgres".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres).
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Postgres connection pool size.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
@@ -77,11 +106,20 @@ async def init_engine(
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
@@ -112,16 +150,25 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
@@ -162,10 +209,10 @@ async def init_engine_from_config(config) -> None:
|
||||
await init_engine("memory")
|
||||
return
|
||||
await init_engine(
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
@@ -29,4 +30,4 @@ class FeedbackRow(Base):
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy import Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
@@ -18,7 +19,7 @@ class LlmCallMetricRow(Base):
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
+7
-6
@@ -44,12 +44,13 @@ def run_migrations_offline() -> None:
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
+8
-6
@@ -7,9 +7,11 @@ Create Date: 2026-05-12
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
|
||||
revision = "20260512_01"
|
||||
@@ -23,9 +25,9 @@ def _has_column(table_name: str, column_name: str) -> bool:
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", sa.JSON(), nullable=True))
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"agents",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(191), nullable=False),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("published", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"tags",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(159), nullable=False),
|
||||
sa.Column("tag_type", sa.String(32), nullable=False),
|
||||
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Create ai_writing_sessions table.
|
||||
|
||||
Revision ID: 20260517_01
|
||||
Revises: 20260515_01
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_01"
|
||||
down_revision = "20260515_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("ai_writing_sessions"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"ai_writing_sessions",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("title", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("user_intent", sa.String(512), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="in_progress"),
|
||||
sa.Column("draft_title", sa.String(512), nullable=True),
|
||||
sa.Column("draft_markdown", sa.Text, nullable=True),
|
||||
sa.Column("completed_interventions", sa.Text, nullable=True),
|
||||
sa.Column("review_result", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_ai_writing_sessions_user_id", "ai_writing_sessions", ["user_id"])
|
||||
op.create_index("ix_ai_writing_sessions_updated_at", "ai_writing_sessions", ["updated_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("ai_writing_sessions"):
|
||||
op.drop_table("ai_writing_sessions")
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Create ai_writing_article_types table with default types.
|
||||
|
||||
Revision ID: 20260517_02
|
||||
Revises: 20260517_01
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_02"
|
||||
down_revision = "20260517_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_DEFAULT_TYPES = [
|
||||
{
|
||||
"key": "news",
|
||||
"label": "新闻资讯",
|
||||
"description": "时效性强的新闻报道,注重事实准确、简洁客观",
|
||||
"prompt_hint": "以新闻报道的风格写作,注重5W1H要素,语言简洁客观,突出时效性和事实准确性",
|
||||
"default_word_count": 600,
|
||||
"sort_order": 0,
|
||||
},
|
||||
{
|
||||
"key": "analysis",
|
||||
"label": "分析报道",
|
||||
"description": "深入分析事件背景与影响,提供专业见解",
|
||||
"prompt_hint": "以分析报道的风格写作,提供深度背景分析、多角度解读和专业见解,逻辑清晰",
|
||||
"default_word_count": 1200,
|
||||
"sort_order": 1,
|
||||
},
|
||||
{
|
||||
"key": "feature",
|
||||
"label": "深度特稿",
|
||||
"description": "人物或事件的深度叙事,注重细节与故事性",
|
||||
"prompt_hint": "以深度特稿的风格写作,注重叙事性和细节描写,人物刻画生动,故事性强,情感真实",
|
||||
"default_word_count": 2000,
|
||||
"sort_order": 2,
|
||||
},
|
||||
{
|
||||
"key": "report",
|
||||
"label": "调研报告",
|
||||
"description": "数据驱动的专题研究报告,结构严谨",
|
||||
"prompt_hint": "以调研报告的风格写作,数据详实,结构严谨(摘要/背景/发现/结论),语言专业规范",
|
||||
"default_word_count": 3000,
|
||||
"sort_order": 3,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("ai_writing_article_types"):
|
||||
op.create_table(
|
||||
"ai_writing_article_types",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("key", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("label", sa.String(128), nullable=False),
|
||||
sa.Column("description", sa.Text, nullable=True),
|
||||
sa.Column("prompt_hint", sa.Text, nullable=True),
|
||||
sa.Column("default_word_count", sa.Integer, nullable=False, server_default="800"),
|
||||
sa.Column("sort_order", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
# 预置默认文章类型(幂等,跳过已存在的 key)。
|
||||
# 用 SQLAlchemy Core 构造而非裸 SQL:``key`` 是 MySQL 保留字,裸
|
||||
# ``SELECT key`` 会触发 1064 语法错误。Core 编译器会按方言自动给保留字
|
||||
# 加引号(MySQL 反引号 / Postgres 双引号),保持跨后端可移植。
|
||||
article_types = sa.table(
|
||||
"ai_writing_article_types",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("key", sa.String),
|
||||
sa.column("label", sa.String),
|
||||
sa.column("description", sa.Text),
|
||||
sa.column("prompt_hint", sa.Text),
|
||||
sa.column("default_word_count", sa.Integer),
|
||||
sa.column("sort_order", sa.Integer),
|
||||
sa.column("created_at", sa.DateTime),
|
||||
sa.column("updated_at", sa.DateTime),
|
||||
)
|
||||
conn = op.get_bind()
|
||||
existing_keys = {row[0] for row in conn.execute(sa.select(article_types.c.key)).fetchall()}
|
||||
now = datetime.now(UTC)
|
||||
for t in _DEFAULT_TYPES:
|
||||
if t["key"] not in existing_keys:
|
||||
conn.execute(
|
||||
article_types.insert().values(
|
||||
id=str(uuid.uuid4()),
|
||||
key=t["key"],
|
||||
label=t["label"],
|
||||
description=t["description"],
|
||||
prompt_hint=t["prompt_hint"],
|
||||
default_word_count=t["default_word_count"],
|
||||
sort_order=t["sort_order"],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("ai_writing_article_types"):
|
||||
op.drop_table("ai_writing_article_types")
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Add transcript column to ai_writing_sessions.
|
||||
|
||||
存储前端组装的完整对话时间线(progressEvents + completedInterventions),
|
||||
供历史会话回显出与实时对话一致的富时间线。
|
||||
|
||||
Revision ID: 20260517_03
|
||||
Revises: 20260517_02
|
||||
Create Date: 2026-05-17
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260517_03"
|
||||
down_revision = "20260517_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
op.add_column("ai_writing_sessions", sa.Column("transcript", sa.Text, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("ai_writing_sessions", "transcript"):
|
||||
op.drop_column("ai_writing_sessions", "transcript")
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Add detail column to skills table.
|
||||
|
||||
Revision ID: 20260518_01
|
||||
Revises: 20260515_01
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_01"
|
||||
down_revision = "20260515_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ``skills`` may not exist yet on a brand-new DB — create_all handles that
|
||||
# case with the column already present, so only patch existing tables.
|
||||
if _has_table("skills") and not _has_column("skills", "detail"):
|
||||
op.add_column("skills", sa.Column("detail", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("skills", "detail"):
|
||||
op.drop_column("skills", "detail")
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Create thread_shares table (per-conversation share links).
|
||||
|
||||
Revision ID: 20260518_02
|
||||
Revises: 20260518_01
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_02"
|
||||
down_revision = "20260518_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
op.create_table(
|
||||
"thread_shares",
|
||||
sa.Column("share_code", sa.String(32), primary_key=True),
|
||||
sa.Column("owner_user_id", sa.String(64), nullable=False),
|
||||
sa.Column("thread_id", sa.String(64), nullable=False),
|
||||
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
elif not _has_column("thread_shares", "thread_id"):
|
||||
# An earlier all-conversations variant of this table — add the column.
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("thread_shares"):
|
||||
op.drop_table("thread_shares")
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"""Ensure thread_shares.thread_id exists (per-conversation share links).
|
||||
|
||||
Standalone migration so databases that already applied an earlier
|
||||
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
|
||||
column) get the column added. Idempotent — a no-op when the column is
|
||||
already present.
|
||||
|
||||
Revision ID: 20260518_03
|
||||
Revises: 20260518_02
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_03"
|
||||
down_revision = "20260518_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
# Fresh database — 20260518_02 already created the table with thread_id.
|
||||
return
|
||||
if not _has_column("thread_shares", "thread_id"):
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Leave the column in place — dropping it would break the share feature.
|
||||
pass
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Add retry_count to scheduled_task_runs.
|
||||
|
||||
Tracks how many recovery attempts a run has consumed so the scheduler can
|
||||
cap retries on stuck/timed-out runs.
|
||||
|
||||
Revision ID: 20260518_04
|
||||
Revises: 20260518_03
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_04"
|
||||
down_revision = "20260518_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.add_column(
|
||||
"scheduled_task_runs",
|
||||
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.drop_column("scheduled_task_runs", "retry_count")
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Add mode to thread_shares.
|
||||
|
||||
Distinguishes "import" shares (recipient copies the conversation into their
|
||||
account) from "view" shares (a public, read-only static page).
|
||||
|
||||
Revision ID: 20260518_05
|
||||
Revises: 20260518_04
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_05"
|
||||
down_revision = "20260518_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return True # table missing entirely — skip (created with the column)
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("thread_shares", "mode"):
|
||||
op.add_column(
|
||||
"thread_shares",
|
||||
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if "thread_shares" in inspector.get_table_names() and any(
|
||||
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
|
||||
):
|
||||
op.drop_column("thread_shares", "mode")
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""Merge the ai-writing and thread-shares migration heads.
|
||||
|
||||
Two independent migration branches diverged from ``20260515_01``: the AI
|
||||
writing feature (``20260517_01..03``) and the thread-shares / scheduled-task
|
||||
work (``20260518_01..05``). Merging the two branches into one repo left Alembic
|
||||
with multiple heads, so ``upgrade head`` failed. This is a no-op merge revision
|
||||
that joins both lineages — it changes no schema.
|
||||
|
||||
Revision ID: 20260518_06
|
||||
Revises: 20260517_03, 20260518_05
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260518_06"
|
||||
down_revision = ("20260517_03", "20260518_05")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
"""Create tool_call_metrics table.
|
||||
|
||||
Revision ID: 20260518_07
|
||||
Revises: 20260518_06
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_07"
|
||||
down_revision = "20260518_06"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("tool_call_metrics"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"tool_call_metrics",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=True),
|
||||
sa.Column("run_id", sa.String(64), nullable=True),
|
||||
sa.Column("agent_name", sa.String(128), nullable=True),
|
||||
sa.Column("tool_name", sa.String(128), nullable=True),
|
||||
sa.Column("skill_name", sa.String(128), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
|
||||
sa.Column("error_category", sa.String(32), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tool_call_metrics_created_at", "tool_call_metrics", ["created_at"])
|
||||
op.create_index("ix_tool_call_metrics_user_id", "tool_call_metrics", ["user_id"])
|
||||
op.create_index("ix_tool_call_metrics_thread_id", "tool_call_metrics", ["thread_id"])
|
||||
op.create_index("ix_tool_call_metrics_run_id", "tool_call_metrics", ["run_id"])
|
||||
op.create_index("ix_tool_call_metrics_agent_name", "tool_call_metrics", ["agent_name"])
|
||||
op.create_index("ix_tool_call_metrics_tool_name", "tool_call_metrics", ["tool_name"])
|
||||
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
|
||||
op.create_index("ix_tool_call_metrics_status", "tool_call_metrics", ["status"])
|
||||
op.create_index("ix_tool_call_metrics_error_category", "tool_call_metrics", ["error_category"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("tool_call_metrics"):
|
||||
op.drop_table("tool_call_metrics")
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Add skill_name column to tool_call_metrics.
|
||||
|
||||
``tool_call_metrics`` is created by ``create_all`` at startup. On deployments
|
||||
where the table was materialised *before* the ``skill_name`` column existed,
|
||||
neither ``create_all`` (it never ALTERs) nor revision ``20260518_07`` (it skips
|
||||
when the table already exists) adds the column — leaving MySQL/Postgres with a
|
||||
schema that the ORM model expects a column it does not have. This revision
|
||||
patches the existing table.
|
||||
|
||||
Revision ID: 20260519_01
|
||||
Revises: 20260518_07
|
||||
Create Date: 2026-05-19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260519_01"
|
||||
down_revision = "20260518_07"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Brand-new DBs: the table is created with the column already present
|
||||
# (by create_all / revision 20260518_07) — nothing to patch.
|
||||
if not _has_table("tool_call_metrics"):
|
||||
return
|
||||
if not _has_column("tool_call_metrics", "skill_name"):
|
||||
op.add_column("tool_call_metrics", sa.Column("skill_name", sa.String(128), nullable=True))
|
||||
if not _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
|
||||
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
|
||||
op.drop_index("ix_tool_call_metrics_skill_name", table_name="tool_call_metrics")
|
||||
if _has_column("tool_call_metrics", "skill_name"):
|
||||
op.drop_column("tool_call_metrics", "skill_name")
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Widen ai_writing_sessions.transcript to LONGTEXT on MySQL.
|
||||
|
||||
The ``transcript`` column stores the full AI-writing conversation timeline as a
|
||||
JSON blob. It was originally created as ``TEXT``, which on MySQL caps at 64 KB —
|
||||
a transcript with many progress events overflows it and the save endpoint
|
||||
returns HTTP 500. This revision widens the column to ``LONGTEXT`` (4 GB) on
|
||||
MySQL. SQLite/PostgreSQL ``TEXT`` is already unbounded, so they are left as-is.
|
||||
|
||||
Revision ID: 20260519_02
|
||||
Revises: 20260519_01
|
||||
Create Date: 2026-05-19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
|
||||
|
||||
revision = "20260519_02"
|
||||
down_revision = "20260519_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return False
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
# Only MySQL has the 64 KB TEXT limit; other backends need no change.
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=LONGTEXT(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if bind.dialect.name != "mysql":
|
||||
return
|
||||
if not _has_column("ai_writing_sessions", "transcript"):
|
||||
return
|
||||
# Revert to TEXT. Rows longer than 64 KB would be truncated, but downgrade
|
||||
# is a developer-only path.
|
||||
import sqlalchemy as sa
|
||||
|
||||
op.alter_column(
|
||||
"ai_writing_sessions",
|
||||
"transcript",
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True,
|
||||
)
|
||||
+4
@@ -32,6 +32,8 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = [
|
||||
@@ -51,5 +53,7 @@ __all__ = [
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"UserRow",
|
||||
]
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunEventRow(Base):
|
||||
@@ -24,9 +25,9 @@ class RunEventRow(Base):
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
# "message" | "trace" | "lifecycle"
|
||||
content: Mapped[str] = mapped_column(Text, default="")
|
||||
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
event_metadata: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
seq: Mapped[int] = mapped_column(nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
|
||||
|
||||
+4
-3
@@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text
|
||||
from sqlalchemy import Boolean, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class NotificationRow(Base):
|
||||
@@ -19,6 +20,6 @@ class NotificationRow(Base):
|
||||
type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class RecommendedQuestionRow(Base):
|
||||
@@ -17,5 +18,5 @@ class RecommendedQuestionRow(Base):
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunRow(Base):
|
||||
@@ -22,8 +23,8 @@ class RunRow(Base):
|
||||
|
||||
model_name: Mapped[str | None] = mapped_column(String(128))
|
||||
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Convenience fields (for listing pages without querying RunEventStore)
|
||||
@@ -43,7 +44,7 @@ class RunRow(Base):
|
||||
# Follow-up association
|
||||
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
|
||||
+34
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
|
||||
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
@abstractmethod
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Return every task across all users. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
"""Delete a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs for a task without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs still in a non-terminal state whose ``started_at`` is older
|
||||
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
|
||||
or were orphaned by a service restart."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
@@ -56,6 +85,11 @@ class ScheduledTaskStore(ABC):
|
||||
async def delete_run(self, run_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
"""Delete a run by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
+43
-2
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
due.sort(key=lambda task: task["next_run_at"])
|
||||
return due[:limit]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
return [dict(task) for task in self._tasks.values()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
task = self._tasks.get(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
task.update(data)
|
||||
return dict(task)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
return self._tasks.pop(task_id, None) is not None
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
|
||||
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
|
||||
return runs[:limit]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
stuck = [
|
||||
dict(run)
|
||||
for run in self._runs.values()
|
||||
if run.get("status") in ("running", "pending")
|
||||
and run.get("started_at") is not None
|
||||
and run["started_at"] <= cutoff
|
||||
]
|
||||
stuck.sort(key=lambda run: run["started_at"])
|
||||
return stuck[:limit]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
key = (data["task_id"], data["scheduled_for"].isoformat())
|
||||
if key in self._runs_by_key:
|
||||
@@ -90,11 +121,21 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["user_id"] != user_id:
|
||||
return False
|
||||
self._runs.pop(run_id, None)
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return False
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
def _drop_run(self, run: dict[str, Any]) -> None:
|
||||
self._runs.pop(run["id"], None)
|
||||
scheduled_for = run.get("scheduled_for")
|
||||
if hasattr(scheduled_for, "isoformat"):
|
||||
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
|
||||
|
||||
+26
-21
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class SchedulerThreadRow(Base):
|
||||
@@ -15,8 +16,8 @@ class SchedulerThreadRow(Base):
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRow(Base):
|
||||
@@ -29,20 +30,20 @@ class ScheduledTaskRow(Base):
|
||||
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
prompt: Mapped[str] = mapped_column(Text)
|
||||
schedule_text: Mapped[str] = mapped_column(String(256))
|
||||
cron_expr: Mapped[str] = mapped_column(String(128))
|
||||
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
last_status: Mapped[str | None] = mapped_column(String(32))
|
||||
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
execution_context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskSubscriptionRow(Base):
|
||||
@@ -53,8 +54,8 @@ class ScheduledTaskSubscriptionRow(Base):
|
||||
task_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
notify_element: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRunRow(Base):
|
||||
@@ -66,21 +67,25 @@ class ScheduledTaskRunRow(Base):
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending")
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
result_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
# Number of recovery attempts made after the first try. A run is retried
|
||||
# when it overruns the per-attempt timeout or is found orphaned in the
|
||||
# "running" state (e.g. after a service restart). See ScheduledTaskService.
|
||||
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskDeliveryProfileRow(Base):
|
||||
__tablename__ = "scheduled_task_delivery_profiles"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(191), index=True)
|
||||
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+71
-1
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
attr = "execution_context_json" if key == "execution_context" else key
|
||||
if hasattr(row, attr):
|
||||
setattr(row, attr, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(ScheduledTaskRunRow.task_id == task_id)
|
||||
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(
|
||||
ScheduledTaskRunRow.status.in_(("running", "pending")),
|
||||
ScheduledTaskRunRow.started_at.is_not(None),
|
||||
ScheduledTaskRunRow.started_at <= cutoff,
|
||||
)
|
||||
.order_by(ScheduledTaskRunRow.started_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = ScheduledTaskRunRow(
|
||||
id=data["id"],
|
||||
@@ -180,6 +234,15 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRunRow, run_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
|
||||
@@ -232,7 +295,14 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRow)
|
||||
.where(ScheduledTaskRow.published.is_(True))
|
||||
.order_by(ScheduledTaskRow.published_at.desc().nullslast(), ScheduledTaskRow.created_at.desc())
|
||||
# ``NULLS LAST`` is unsupported by MySQL — sort by an
|
||||
# ``IS NULL`` flag instead (False<True ⇒ non-null rows first),
|
||||
# which is portable across mysql/sqlite/postgres.
|
||||
.order_by(
|
||||
ScheduledTaskRow.published_at.is_(None),
|
||||
ScheduledTaskRow.published_at.desc(),
|
||||
ScheduledTaskRow.created_at.desc(),
|
||||
)
|
||||
.offset(max(0, offset))
|
||||
.limit(max(1, limit))
|
||||
)
|
||||
|
||||
+6
@@ -69,6 +69,7 @@ class MemorySkillStore(SkillStore):
|
||||
"name": name,
|
||||
"owner_user_id": data.get("owner_user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"detail": data.get("detail"),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -83,6 +84,7 @@ class MemorySkillStore(SkillStore):
|
||||
"name": name,
|
||||
"owner_user_id": data.get("owner_user_id"),
|
||||
"published": bool(data.get("published", True)),
|
||||
"detail": data.get("detail"),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -98,6 +100,8 @@ class MemorySkillStore(SkillStore):
|
||||
return None
|
||||
if "published" in data:
|
||||
record["published"] = bool(data["published"])
|
||||
if "detail" in data:
|
||||
record["detail"] = data["detail"]
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
@@ -109,6 +113,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["published"] = bool(data["published"])
|
||||
if "owner_user_id" in data:
|
||||
record["owner_user_id"] = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
record["detail"] = data["detail"]
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
+6
-1
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String
|
||||
from sqlalchemy import Boolean, DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,5 +24,10 @@ class SkillRow(Base):
|
||||
name: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Human-authored long-form detail/introduction, editable by the skill owner
|
||||
# or an admin. Distinct from the skill ``description`` parsed from SKILL.md;
|
||||
# kept in the DB so admins can annotate built-in skills without rewriting
|
||||
# the shared on-disk SKILL.md file.
|
||||
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -63,6 +63,7 @@ class SkillRepository(SkillStore):
|
||||
name=data["name"],
|
||||
owner_user_id=data.get("owner_user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
detail=data.get("detail"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -81,6 +82,7 @@ class SkillRepository(SkillStore):
|
||||
name=data["name"],
|
||||
owner_user_id=data.get("owner_user_id"),
|
||||
published=bool(data.get("published", True)),
|
||||
detail=data.get("detail"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -101,6 +103,8 @@ class SkillRepository(SkillStore):
|
||||
return None
|
||||
if "published" in data:
|
||||
row.published = bool(data["published"])
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
@@ -115,6 +119,8 @@ class SkillRepository(SkillStore):
|
||||
row.published = bool(data["published"])
|
||||
if "owner_user_id" in data:
|
||||
row.owner_user_id = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
+6
-5
@@ -14,10 +14,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, UniqueConstraint
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
# Allowed tag/target types. Kept here so persistence + API share one source.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
|
||||
@@ -28,12 +29,12 @@ class TagRow(Base):
|
||||
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(159), nullable=False)
|
||||
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
@@ -44,4 +45,4 @@ class TagAssignmentRow(Base):
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, String
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class ThreadMetaRow(Base):
|
||||
@@ -18,6 +19,6 @@ class ThreadMetaRow(Base):
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Persistence layer for conversation-history share links."""
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
|
||||
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
|
||||
|
||||
__all__ = [
|
||||
"ThreadShareStore",
|
||||
"MemoryThreadShareStore",
|
||||
"ThreadShareRepository",
|
||||
"generate_share_code",
|
||||
"make_thread_share_store",
|
||||
]
|
||||
|
||||
|
||||
def make_thread_share_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryThreadShareStore()
|
||||
return ThreadShareRepository(session_factory)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Abstract persistence API for per-conversation share links."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
|
||||
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
|
||||
|
||||
|
||||
def generate_share_code(length: int = 8) -> str:
|
||||
"""Return a random, unguessable, human-readable share code."""
|
||||
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
|
||||
|
||||
|
||||
class ThreadShareStore(ABC):
|
||||
@abstractmethod
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
"""Create a new share link for one conversation and return its record.
|
||||
|
||||
``mode`` is ``"import"`` (recipient copies the conversation) or
|
||||
``"view"`` (a public read-only static page).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
"""Look up a share by code regardless of owner (None if missing)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
|
||||
|
||||
Used so re-sharing the same conversation in the same mode reuses its
|
||||
existing code instead of minting a new one each time.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
"""List all share links created by ``owner_user_id`` (newest first)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
"""Revoke a share link. Returns False if missing or not owned."""
|
||||
raise NotImplementedError
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
|
||||
|
||||
class MemoryThreadShareStore(ThreadShareStore):
|
||||
def __init__(self) -> None:
|
||||
self._shares: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
code = generate_share_code()
|
||||
while code in self._shares:
|
||||
code = generate_share_code()
|
||||
record = {
|
||||
"share_code": code,
|
||||
"owner_user_id": owner_user_id,
|
||||
"thread_id": thread_id,
|
||||
"revoked": False,
|
||||
"mode": mode,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._shares[code] = record
|
||||
return copy.deepcopy(record)
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
record = self._shares.get(share_code)
|
||||
return copy.deepcopy(record) if record is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
matches = [
|
||||
r
|
||||
for r in self._shares.values()
|
||||
if r["owner_user_id"] == owner_user_id
|
||||
and r["thread_id"] == thread_id
|
||||
and r.get("mode", "import") == mode
|
||||
and not r["revoked"]
|
||||
]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
|
||||
rows.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
record = self._shares.get(share_code)
|
||||
if record is None or record["owner_user_id"] != owner_user_id:
|
||||
return False
|
||||
record["revoked"] = True
|
||||
return True
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""ORM model for conversation share links.
|
||||
|
||||
A share link is a short, revocable code created by a user for *one* of their
|
||||
conversations. Redeeming it copies that single conversation into the redeemer's
|
||||
account. The table only stores the link itself — the conversation is copied
|
||||
lazily at import time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ThreadShareRow(Base):
|
||||
__tablename__ = "thread_shares"
|
||||
|
||||
# Short, URL-safe, unguessable code embedded in the share link.
|
||||
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# The user who created the share.
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# The single conversation (thread) this link shares.
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Soft-disable: a revoked link can no longer be previewed or imported.
|
||||
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
# Share mode:
|
||||
# "import" — recipient (a logged-in user) copies the conversation into
|
||||
# their own account and can keep chatting in the copy.
|
||||
# "view" — a public, read-only static page anyone can open without
|
||||
# logging in; viewers cannot send new messages.
|
||||
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""SQLAlchemy-backed per-conversation share-link repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
|
||||
|
||||
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
created = data.get("created_at")
|
||||
if isinstance(created, datetime):
|
||||
data["created_at"] = created.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class ThreadShareRepository(ThreadShareStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
# Retry on the (astronomically unlikely) code collision.
|
||||
async with self._sf() as session:
|
||||
for _ in range(8):
|
||||
code = generate_share_code()
|
||||
if await session.get(ThreadShareRow, code) is not None:
|
||||
continue
|
||||
row = ThreadShareRow(
|
||||
share_code=code,
|
||||
owner_user_id=owner_user_id,
|
||||
thread_id=thread_id,
|
||||
revoked=False,
|
||||
mode=mode,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
raise RuntimeError("Failed to allocate a unique share code")
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
stmt = (
|
||||
select(ThreadShareRow)
|
||||
.where(
|
||||
ThreadShareRow.owner_user_id == owner_user_id,
|
||||
ThreadShareRow.thread_id == thread_id,
|
||||
ThreadShareRow.mode == mode,
|
||||
ThreadShareRow.revoked.is_(False),
|
||||
)
|
||||
.order_by(ThreadShareRow.created_at.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
if row is None or row.owner_user_id != owner_user_id:
|
||||
return False
|
||||
row.revoked = True
|
||||
await session.commit()
|
||||
return True
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""Persistence layer for per-tool-call (skill-call) metrics.
|
||||
|
||||
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
|
||||
writes one row recording duration and outcome. Failures are bucketed into a
|
||||
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
|
||||
network_error / not_found / empty_result / tool_error) so administrators can
|
||||
spot retrieval outages, blocked IPs, and quota problems at a glance.
|
||||
|
||||
**Skill-call merging**: a skill is often touched by several tool calls inside
|
||||
one run (read ``SKILL.md``, run a script, retry after a failure). Rather than
|
||||
one row per call, ``SqlToolMetricsStore.record`` upserts a single row per
|
||||
``(run_id, skill_name)``: any success in the run wins (a skill that failed then
|
||||
finally succeeded is recorded as one success), and a skill that only ever
|
||||
failed yields one ``error`` row whose ``error_message`` accumulates every
|
||||
attempt's message. Non-skill tool calls keep one row each.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"ToolCallMetricRow",
|
||||
"ToolMetricRecord",
|
||||
"ToolMetricsStore",
|
||||
"SqlToolMetricsStore",
|
||||
"make_tool_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlToolMetricsStore(session_factory)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
"""Abstract repository contract for tool / skill call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricRecord:
|
||||
"""In-memory shape of a tool-call metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
tool_name: str | None = None
|
||||
# The specific Agent Skill this call targets, when the tool is a skill tool
|
||||
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
|
||||
# call's ``name`` argument. Null for every non-skill tool call. This is
|
||||
# what lets the admin see "skill X was invoked, and succeeded/failed".
|
||||
skill_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
status: str = "success"
|
||||
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
|
||||
# network_error / not_found / empty_result / tool_error. Null on success.
|
||||
error_category: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
tool_name: str | None = None
|
||||
# When True, keep only rows that target a specific Agent Skill
|
||||
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
|
||||
skill_only: bool = False
|
||||
# Filter to one specific skill by name.
|
||||
skill_name: str | None = None
|
||||
status: str | None = None
|
||||
error_category: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class ToolMetricsStore(ABC):
|
||||
"""Persistence contract for the tool-metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: ToolMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
|
||||
|
||||
Only rows with a non-null ``skill_name`` are counted, so this answers
|
||||
"which skills were invoked, and how often did each succeed/fail".
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""ORM model for per-tool-call (skill-call) metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ToolCallMetricRow(Base):
|
||||
"""One row per completed (or failed) tool / skill invocation.
|
||||
|
||||
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
|
||||
failures — search tools that swallow an HTTP 429/403 and return an error
|
||||
payload instead of raising — so an admin can see "retrieval failed" /
|
||||
"IP blocked" events that never surfaced as exceptions.
|
||||
"""
|
||||
|
||||
__tablename__ = "tool_call_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# The specific Agent Skill targeted by a skill tool call (from the call's
|
||||
# ``name`` argument); null for non-skill tool calls.
|
||||
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
# "success" or "error".
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
# Coarse failure bucket; null on success.
|
||||
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"tool_name": self.tool_name,
|
||||
"skill_name": self.skill_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"status": self.status,
|
||||
"error_category": self.error_category,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
"""SQLAlchemy-backed tool metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Separator joining every failed attempt's message inside one merged skill row.
|
||||
_ERROR_SEP = "\n--- \n"
|
||||
# Hard cap on the accumulated ``error_message`` of a merged row.
|
||||
_MERGED_ERROR_LIMIT = 2000
|
||||
|
||||
# A single skill invocation often spans several tool calls (read SKILL.md, run
|
||||
# a script, ...), and the model may retry a skill within the same run. Those
|
||||
# fire-and-forget writes can interleave on the event loop, so the read → merge →
|
||||
# write of a skill row is guarded by a lock. ``asyncio.Lock`` binds to the loop
|
||||
# it is used on; the lock is therefore (re)created whenever the running loop
|
||||
# changes (the Gateway keeps one persistent loop, while the ``asyncio.run``
|
||||
# fallback path spins up a throwaway loop per call — already serialized).
|
||||
_merge_lock: asyncio.Lock | None = None
|
||||
_merge_lock_loop: object = None
|
||||
|
||||
|
||||
def _skill_merge_lock() -> asyncio.Lock:
|
||||
global _merge_lock, _merge_lock_loop
|
||||
loop = asyncio.get_running_loop()
|
||||
if _merge_lock is None or _merge_lock_loop is not loop:
|
||||
_merge_lock = asyncio.Lock()
|
||||
_merge_lock_loop = loop
|
||||
return _merge_lock
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _merge_error_messages(existing: str | None, new: str | None) -> str | None:
|
||||
"""Append a failed attempt's message to the accumulated error text.
|
||||
|
||||
All failures are kept ("都统计下来"), separated and capped at
|
||||
:data:`_MERGED_ERROR_LIMIT` characters.
|
||||
"""
|
||||
new = (new or "").strip()
|
||||
if not new:
|
||||
return existing
|
||||
existing = (existing or "").strip()
|
||||
if not existing:
|
||||
return new
|
||||
if new in existing:
|
||||
# Identical retry failure — no point storing the same line twice.
|
||||
return existing
|
||||
combined = existing + _ERROR_SEP + new
|
||||
if len(combined) > _MERGED_ERROR_LIMIT:
|
||||
combined = combined[: _MERGED_ERROR_LIMIT - 3] + "..."
|
||||
return combined
|
||||
|
||||
|
||||
def _merge_skill_row(existing: ToolCallMetricRow, metric: ToolMetricRecord) -> None:
|
||||
"""Fold another same-run, same-skill call into an existing metric row.
|
||||
|
||||
Outcome rules:
|
||||
- any success in the run -> the row becomes a single ``success`` and earlier
|
||||
failures are dropped ("先报错、最后成功 -> 记一次成功");
|
||||
- all calls failed -> a single ``error`` row whose ``error_message``
|
||||
accumulates every attempt's message ("一直报错 -> 记一次,报错信息都统计下来").
|
||||
"""
|
||||
existing.duration_ms = (existing.duration_ms or 0) + (metric.duration_ms or 0)
|
||||
if metric.status == "success":
|
||||
if existing.status != "success":
|
||||
existing.status = "success"
|
||||
existing.error_category = None
|
||||
existing.error_message = None
|
||||
return
|
||||
# metric.status == "error"
|
||||
if existing.status == "success":
|
||||
# The skill already succeeded once this run — keep that outcome.
|
||||
return
|
||||
existing.status = "error"
|
||||
if metric.error_category:
|
||||
existing.error_category = metric.error_category
|
||||
existing.error_message = _merge_error_messages(existing.error_message, metric.error_message)
|
||||
|
||||
|
||||
def _row_from_metric(metric: ToolMetricRecord) -> ToolCallMetricRow:
|
||||
return ToolCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
tool_name=metric.tool_name,
|
||||
skill_name=metric.skill_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
status=metric.status,
|
||||
error_category=metric.error_category,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
|
||||
|
||||
class SqlToolMetricsStore(ToolMetricsStore):
|
||||
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
# Skill calls are merged per run: the model may touch one skill across
|
||||
# several tool calls / retries, and the admin wants a single row per
|
||||
# (run, skill) instead of one per call. Non-skill tool calls, and skill
|
||||
# calls with no run_id, fall back to a plain insert.
|
||||
if metric.skill_name and metric.run_id:
|
||||
await self._record_skill_merged(metric)
|
||||
return
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(_row_from_metric(metric))
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort; never surface a write failure to the runtime.
|
||||
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
async def _record_skill_merged(self, metric: ToolMetricRecord) -> None:
|
||||
"""Upsert a per-(run, skill) row, folding repeated calls into one."""
|
||||
try:
|
||||
async with _skill_merge_lock():
|
||||
async with self._sf() as session:
|
||||
existing = await session.scalar(
|
||||
select(ToolCallMetricRow)
|
||||
.where(
|
||||
ToolCallMetricRow.run_id == metric.run_id,
|
||||
ToolCallMetricRow.skill_name == metric.skill_name,
|
||||
)
|
||||
.order_by(ToolCallMetricRow.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if existing is None:
|
||||
session.add(_row_from_metric(metric))
|
||||
else:
|
||||
_merge_skill_row(existing, metric)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to merge tool_call_metrics for skill %s (run %s)",
|
||||
metric.skill_name,
|
||||
metric.run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _apply_filters(self, stmt, query: ToolMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
|
||||
if query.tool_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
|
||||
if query.skill_only:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
|
||||
if query.skill_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(ToolCallMetricRow.status == query.status)
|
||||
if query.error_category:
|
||||
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row.to_dict() for row in result.scalars()]
|
||||
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page = await self.list(
|
||||
ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
tool_name=query.tool_name,
|
||||
skill_only=query.skill_only,
|
||||
skill_name=query.skill_name,
|
||||
status=query.status,
|
||||
error_category=query.error_category,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
# Count success/error per skill. Force the skill-only filter so rows
|
||||
# without a skill_name (plain tool calls) never leak into the result.
|
||||
scoped = ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
skill_only=True,
|
||||
skill_name=query.skill_name,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
)
|
||||
stmt = select(
|
||||
ToolCallMetricRow.skill_name,
|
||||
ToolCallMetricRow.status,
|
||||
func.count().label("n"),
|
||||
)
|
||||
stmt = self._apply_filters(stmt, scoped).group_by(
|
||||
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
|
||||
)
|
||||
agg: dict[str, dict[str, int]] = {}
|
||||
async with self._sf() as session:
|
||||
for skill_name, status, n in (await session.execute(stmt)).all():
|
||||
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
|
||||
bucket["total"] += n
|
||||
bucket["success" if status == "success" else "error"] += n
|
||||
return [
|
||||
{"skill_name": name, **counts}
|
||||
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
|
||||
]
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Portable SQLAlchemy column types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
|
||||
# fixed offset is exact and avoids depending on the `tzdata` package (which the
|
||||
# stdlib `zoneinfo` needs on Windows / minimal containers).
|
||||
BEIJING_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
class PortableJSON(TypeDecorator):
|
||||
"""Use native JSON where available, and TEXT on older MySQL servers."""
|
||||
|
||||
impl = JSON
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(Text())
|
||||
return dialect.type_descriptor(JSON())
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None:
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None or not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class PortableLongText(TypeDecorator):
|
||||
"""A large text column.
|
||||
|
||||
MySQL's ``TEXT`` caps at 64 KB, which silently truncates — or, in STRICT
|
||||
mode, raises ``Data too long`` (HTTP 500) — for large JSON blobs such as the
|
||||
AI-writing conversation transcript. This decorator maps to MySQL ``LONGTEXT``
|
||||
(4 GB) while staying plain ``TEXT`` on SQLite/PostgreSQL, where ``TEXT`` is
|
||||
already unbounded.
|
||||
"""
|
||||
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(LONGTEXT())
|
||||
return dialect.type_descriptor(Text())
|
||||
|
||||
|
||||
class BeijingDateTime(TypeDecorator):
|
||||
"""Store timestamps as Beijing (UTC+8) wall-clock time.
|
||||
|
||||
The application works in UTC internally (``datetime.now(UTC)`` everywhere).
|
||||
MySQL ``DATETIME`` and SQLite have no timezone, so the driver serializes the
|
||||
wall-clock fields of whatever datetime it is given and drops ``tzinfo``.
|
||||
Without conversion a row stores the UTC wall clock and an operator browsing
|
||||
the table sees a time 8 hours behind Beijing.
|
||||
|
||||
This decorator converts UTC-aware values to Beijing on the way in (stored as
|
||||
naive Beijing wall-clock) and re-attaches the Beijing zone on the way out, so
|
||||
the database holds Beijing time while Python keeps receiving correct
|
||||
timezone-aware datetimes — instant comparisons stay correct either way.
|
||||
|
||||
PostgreSQL has a real ``TIMESTAMP WITH TIME ZONE`` type and stores the
|
||||
instant unambiguously, so values pass through unchanged there.
|
||||
"""
|
||||
|
||||
impl = DateTime
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
# timezone=True keeps PostgreSQL on timestamptz; MySQL/SQLite ignore it.
|
||||
return dialect.type_descriptor(DateTime(timezone=True))
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Naive values follow the codebase convention of being UTC.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Stored as naive Beijing wall-clock; re-attach the zone.
|
||||
return value.replace(tzinfo=BEIJING_TZ)
|
||||
return value.astimezone(BEIJING_TZ)
|
||||
+4
-3
@@ -13,10 +13,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy import Boolean, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
@@ -25,7 +26,7 @@ class UserRow(Base):
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
@@ -33,7 +34,7 @@ class UserRow(Base):
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
+1
@@ -87,6 +87,7 @@ class RunEventStore(abc.ABC):
|
||||
limit: int = 50,
|
||||
before_seq: int | None = None,
|
||||
after_seq: int | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Return displayable messages (category=message) for a specific run, ordered by seq ascending.
|
||||
|
||||
|
||||
+3
-1
@@ -152,7 +152,9 @@ class JsonlRunEventStore(RunEventStore):
|
||||
events = [e for e in events if e.get("event_type") in event_types]
|
||||
return events[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# JSONL events carry no per-event owner; user_id is accepted for
|
||||
# interface parity with DbRunEventStore but ignored.
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
filtered = [e for e in events if e.get("category") == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
+3
-1
@@ -97,7 +97,9 @@ class MemoryRunEventStore(RunEventStore):
|
||||
filtered = [e for e in filtered if e["event_type"] in event_types]
|
||||
return filtered[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# The in-memory store keeps no per-event owner, so user_id is accepted
|
||||
# for interface parity with DbRunEventStore but ignored.
|
||||
all_events = self._events.get(thread_id, [])
|
||||
filtered = [e for e in all_events if e["run_id"] == run_id and e["category"] == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
Loaded 100 of 476 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user