初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,37 @@
|
||||
"""DeerFlow 记忆模块 (Memory V2)。
|
||||
|
||||
架构 (详见 docs/MEMORY_V2_DESIGN_ZH.md):
|
||||
- :class:`MemoryProvider` 可插拔记忆 provider 抽象基类
|
||||
- :class:`MemoryManager` 编排内建 + 至多 1 个外部 provider
|
||||
- :func:`build_memory_manager` 按生效配置组装 manager 的统一入口
|
||||
- :class:`BuiltinFileProvider` 本地 USER.md / MEMORY.md provider
|
||||
- :class:`HindsightProvider` Hindsight HTTP 外部 provider
|
||||
- :mod:`deerflow.agents.memory.security` 记忆内容安全扫描
|
||||
|
||||
记忆有两条自动写入路径:LLM 主动调 ``memory`` / ``hindsight_*`` 工具,
|
||||
以及 Hindsight 的 auto-retain。注入走系统提示词 (builtin frozen snapshot)
|
||||
与 ``MemoryMiddleware.wrap_model_call`` (Hindsight 动态召回)。
|
||||
"""
|
||||
|
||||
from deerflow.agents.memory.manager import MemoryManager, build_memory_manager
|
||||
from deerflow.agents.memory.message_processing import filter_messages_for_memory
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
from deerflow.agents.memory.security import (
|
||||
MemoryContentBlocked,
|
||||
assert_memory_content_safe,
|
||||
scan_memory_content,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MemoryProvider",
|
||||
"MemoryManager",
|
||||
"build_memory_manager",
|
||||
"BuiltinFileProvider",
|
||||
"HindsightProvider",
|
||||
"MemoryContentBlocked",
|
||||
"scan_memory_content",
|
||||
"assert_memory_content_safe",
|
||||
"filter_messages_for_memory",
|
||||
]
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Memory V2 - MemoryManager 编排层。
|
||||
|
||||
职责
|
||||
====
|
||||
编排 1 个内建 provider (``builtin``,始终存在) + 至多 1 个外部 provider
|
||||
(P2 期的 ``hindsight``)。把系统提示词组装、召回、生命周期 hook 分发到各 provider。
|
||||
任一 provider 出错都不会阻断其他 provider,也不会阻断对话。
|
||||
|
||||
典型用法 (由 MemoryMiddleware 每轮构建)
|
||||
=======================================
|
||||
::
|
||||
|
||||
manager = MemoryManager()
|
||||
manager.add_provider(BuiltinFileProvider(...))
|
||||
manager.add_provider(hindsight_provider) # 可选,至多 1 个
|
||||
manager.initialize_all(user_id=..., agent_id=..., thread_id=..., base_dir=...)
|
||||
|
||||
block = manager.build_injection_block(query=user_msg, thread_id=tid)
|
||||
# ... 把 block 注入系统提示词 ...
|
||||
|
||||
manager.sync_all(user_msg, assistant_msg, thread_id=tid)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 内建 provider 的固定名字。
|
||||
BUILTIN_PROVIDER_NAME = "builtin"
|
||||
KNOWLEDGE_PROVIDER_NAME = "knowledge"
|
||||
_INTERNAL_PROVIDER_NAMES = {BUILTIN_PROVIDER_NAME, KNOWLEDGE_PROVIDER_NAME}
|
||||
|
||||
|
||||
class MemoryManager:
|
||||
"""编排内建 provider + 至多 1 个外部 provider。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._providers: list[MemoryProvider] = []
|
||||
self._has_external: bool = False
|
||||
|
||||
# ---- 注册 ------------------------------------------------------------
|
||||
|
||||
def add_provider(self, provider: MemoryProvider) -> bool:
|
||||
"""注册一个 provider。
|
||||
|
||||
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
|
||||
第二个会被拒绝并 log 警告。返回是否注册成功。
|
||||
"""
|
||||
is_internal = provider.name in _INTERNAL_PROVIDER_NAMES
|
||||
if not is_internal:
|
||||
if self._has_external:
|
||||
existing = next(
|
||||
(p.name for p in self._providers if p.name not in _INTERNAL_PROVIDER_NAMES),
|
||||
"unknown",
|
||||
)
|
||||
logger.warning(
|
||||
"拒绝注册记忆 provider '%s' —— 已存在外部 provider '%s'。"
|
||||
"同一时间只允许 1 个外部 provider,请用 config.memory.provider 选择。",
|
||||
provider.name,
|
||||
existing,
|
||||
)
|
||||
return False
|
||||
self._has_external = True
|
||||
self._providers.append(provider)
|
||||
logger.info("记忆 provider '%s' 已注册", provider.name)
|
||||
return True
|
||||
|
||||
@property
|
||||
def providers(self) -> list[MemoryProvider]:
|
||||
"""按注册顺序返回所有 provider。"""
|
||||
return list(self._providers)
|
||||
|
||||
def get_provider(self, name: str) -> MemoryProvider | None:
|
||||
"""按名字取 provider,不存在返回 ``None``。"""
|
||||
for p in self._providers:
|
||||
if p.name == name:
|
||||
return p
|
||||
return None
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize_all(self, **kwargs: Any) -> None:
|
||||
"""初始化所有 provider。单个 provider 失败不影响其他。"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.initialize(**kwargs)
|
||||
except Exception as exc: # noqa: BLE001 - provider 失败必须隔离
|
||||
logger.warning("记忆 provider '%s' initialize 失败:%s", provider.name, exc)
|
||||
|
||||
def shutdown_all(self) -> None:
|
||||
"""关闭所有 provider (逆序,保证干净拆除)。"""
|
||||
for provider in reversed(self._providers):
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' shutdown 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 系统提示词组装 / 召回 -------------------------------------------
|
||||
|
||||
def build_system_prompt(self) -> str:
|
||||
"""收集所有 provider 的静态系统提示词块。"""
|
||||
blocks: list[str] = []
|
||||
for provider in self._providers:
|
||||
try:
|
||||
block = provider.system_prompt_block()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' system_prompt_block 失败:%s", provider.name, exc)
|
||||
continue
|
||||
if block and block.strip():
|
||||
blocks.append(block)
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
def prefetch_all(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""收集所有 provider 的召回上下文。单个失败不阻断其他。"""
|
||||
parts: list[str] = []
|
||||
for provider in self._providers:
|
||||
try:
|
||||
result = provider.prefetch(query, thread_id=thread_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' prefetch 失败 (非致命):%s", provider.name, exc)
|
||||
continue
|
||||
if result and result.strip():
|
||||
parts.append(result)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def build_injection_block(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""组装注入系统提示词的完整记忆块。
|
||||
|
||||
= 所有 provider 的静态块 (frozen snapshot 等) + 动态召回块。
|
||||
返回空串表示无内容可注入。
|
||||
"""
|
||||
static_part = self.build_system_prompt()
|
||||
dynamic_part = self.prefetch_all(query, thread_id=thread_id)
|
||||
parts = [p for p in (static_part, dynamic_part) if p and p.strip()]
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def queue_prefetch_all(self, query: str, *, thread_id: str = "") -> None:
|
||||
"""通知所有 provider 后台预热下一轮召回。"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.queue_prefetch(query, thread_id=thread_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' queue_prefetch 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 写入 ------------------------------------------------------------
|
||||
|
||||
def sync_all(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""把一轮完成的对话同步到所有 provider。
|
||||
|
||||
``turn_index`` 是当前对话的轮次序号,透传给各 provider 用于轮次节流。
|
||||
"""
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.sync_turn(
|
||||
user_content,
|
||||
assistant_content,
|
||||
thread_id=thread_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("记忆 provider '%s' sync_turn 失败:%s", provider.name, exc)
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入后,通知外部 provider 镜像。
|
||||
|
||||
跳过 builtin 自身 (它就是写入源,镜像回去没有意义)。
|
||||
"""
|
||||
for provider in self._providers:
|
||||
if provider.name == BUILTIN_PROVIDER_NAME:
|
||||
continue
|
||||
try:
|
||||
provider.on_memory_write(action, target, content, metadata)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' on_memory_write 失败:%s", provider.name, exc)
|
||||
|
||||
# ---- 可选 hooks ------------------------------------------------------
|
||||
|
||||
def on_session_switch_all(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""通知所有 provider thread_id 已切换。"""
|
||||
if not new_thread_id:
|
||||
return
|
||||
for provider in self._providers:
|
||||
try:
|
||||
provider.on_session_switch(
|
||||
new_thread_id,
|
||||
parent_thread_id=parent_thread_id,
|
||||
reset=reset,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("记忆 provider '%s' on_session_switch 失败:%s", provider.name, exc)
|
||||
|
||||
|
||||
def build_memory_manager(
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
config: Any = None,
|
||||
app_config: Any = None,
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
这是构建 manager 的统一入口 —— 根据 ``config.version`` 选择 V1 或 V2 provider:
|
||||
|
||||
- ``version="v1"``: 仅注册 V1BuiltinProvider (memory.json + LLM 自动提取)
|
||||
- ``version="v2"``: 注册 BuiltinFileProvider + 可选 HindsightProvider (USER.md/MEMORY.md)
|
||||
|
||||
所有 provider 已 ``initialize`` 完毕,可直接使用。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
user_id / agent_id / thread_id:
|
||||
请求上下文标识。
|
||||
config:
|
||||
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
|
||||
"""
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
if config is None:
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = MemoryManager()
|
||||
base_dir = str(get_paths().base_dir)
|
||||
|
||||
memory_enabled = bool(getattr(config, "enabled", True) and getattr(config.injection, "enabled", True))
|
||||
|
||||
if memory_enabled and getattr(config, "version", "v2") == "v1":
|
||||
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
v1_cfg = config.v1
|
||||
manager.add_provider(
|
||||
V1BuiltinProvider(
|
||||
max_facts=v1_cfg.max_facts,
|
||||
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
|
||||
debounce_seconds=v1_cfg.debounce_seconds,
|
||||
model_name=v1_cfg.model_name,
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
)
|
||||
elif memory_enabled:
|
||||
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
if config.builtin.enabled:
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
)
|
||||
)
|
||||
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
try:
|
||||
from deerflow.agents.memory.providers.knowledge import KnowledgeProvider
|
||||
|
||||
knowledge = KnowledgeProvider(app_config=app_config)
|
||||
if knowledge.is_available():
|
||||
manager.add_provider(knowledge)
|
||||
except Exception as exc: # noqa: BLE001 - 知识库路由失败不能阻断对话
|
||||
logger.warning("加载 Knowledge provider 失败:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
base_dir=base_dir,
|
||||
app_config=app_config,
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
__all__ = ["MemoryManager", "BUILTIN_PROVIDER_NAME", "build_memory_manager"]
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
"""把对话整理成记忆写入所需输入的共享辅助函数。
|
||||
|
||||
Memory V2 说明:后台抽取已舍弃,这里只保留 ``filter_messages_for_memory``
|
||||
(供 MemoryMiddleware 取出最后一轮对话用于 Hindsight retain)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from copy import copy
|
||||
from typing import Any
|
||||
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", re.IGNORECASE)
|
||||
|
||||
|
||||
def extract_message_text(message: Any) -> str:
|
||||
"""Extract plain text from message content for filtering and signal detection."""
|
||||
content = getattr(message, "content", "")
|
||||
if isinstance(content, list):
|
||||
text_parts: list[str] = []
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
text_parts.append(part)
|
||||
elif isinstance(part, dict):
|
||||
text_val = part.get("text")
|
||||
if isinstance(text_val, str):
|
||||
text_parts.append(text_val)
|
||||
return " ".join(text_parts)
|
||||
return str(content)
|
||||
|
||||
|
||||
def count_user_turns(filtered_messages: list[Any]) -> int:
|
||||
"""统计已过滤消息里的用户轮次数 (= human 消息条数)。
|
||||
|
||||
供 MemoryMiddleware 算出当前轮次序号传给 provider 做轮次节流 ——
|
||||
provider 每轮被重建、无法靠自身计数,必须由这里按对话历史推算。
|
||||
入参应是 :func:`filter_messages_for_memory` 的输出。
|
||||
"""
|
||||
return sum(1 for msg in filtered_messages if getattr(msg, "type", None) == "human")
|
||||
|
||||
|
||||
def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
"""Keep only user inputs and final assistant responses for memory updates."""
|
||||
filtered = []
|
||||
skip_next_ai = False
|
||||
for msg in messages:
|
||||
msg_type = getattr(msg, "type", None)
|
||||
|
||||
if msg_type == "human":
|
||||
content_str = extract_message_text(msg)
|
||||
if "<uploaded_files>" in content_str:
|
||||
stripped = _UPLOAD_BLOCK_RE.sub("", content_str).strip()
|
||||
if not stripped:
|
||||
skip_next_ai = True
|
||||
continue
|
||||
clean_msg = copy(msg)
|
||||
clean_msg.content = stripped
|
||||
filtered.append(clean_msg)
|
||||
skip_next_ai = False
|
||||
else:
|
||||
filtered.append(msg)
|
||||
skip_next_ai = False
|
||||
elif msg_type == "ai":
|
||||
tool_calls = getattr(msg, "tool_calls", None)
|
||||
if not tool_calls:
|
||||
if skip_next_ai:
|
||||
skip_next_ai = False
|
||||
continue
|
||||
filtered.append(msg)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
_CORRECTION_PATTERNS = re.compile(
|
||||
r"\b(?:no[,.]?\s+(?:that'?s?\s+)?(?:wrong|incorrect|not\s+right|not\s+what)|"
|
||||
r"actually[,.]?\s+(?:it\s+(?:is|should)|you\s+(?:should|need|must)|the\s+correct)|"
|
||||
r"that'?s?\s+(?:wrong|incorrect|not\s+right|not\s+what\s+i)|"
|
||||
r"you\s+(?:got\s+it\s+wrong|misunderstood|made\s+an?\s+error|made\s+a\s+mistake)|"
|
||||
r"incorrect[,.]|wrong[,.]|mistake[,.]|correction:|that\s+is\s+not\s+right)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_REINFORCEMENT_PATTERNS = re.compile(
|
||||
r"\b(?:perfect[,.]?|exactly[,.]?|correct[,.]?|that'?s?\s+(?:right|correct|perfect|exactly\s+right)|"
|
||||
r"you\s+(?:got\s+it|nailed\s+it)|great\s+(?:job|work)|well\s+done|"
|
||||
r"yes[,.]?\s+(?:that'?s?\s+(?:right|correct|exactly)|exactly)|"
|
||||
r"keep\s+(?:it\s+up|doing\s+that|going)|"
|
||||
r"(?:that'?s?\s+|this\s+is\s+)?(?:exactly|precisely)\s+(?:what\s+i|right))\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def detect_correction(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains explicit user correction signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _CORRECTION_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_reinforcement(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains positive reinforcement signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _REINFORCEMENT_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Memory V2 - 记忆 Provider 抽象基类。
|
||||
|
||||
设计动机
|
||||
========
|
||||
Hermes Agent 的可插拔记忆系统的 deerflow 版本。所有具体的记忆后端
|
||||
(builtin 本地文件、Hindsight HTTP 等)都实现这个接口,由 ``MemoryManager``
|
||||
统一编排。Provider 之间互不感知,可自由组合 (内建 + 至多 1 个外部 plugin)。
|
||||
|
||||
生命周期
|
||||
========
|
||||
:func:`MemoryProvider.initialize` 初始化 (创建 bank、加载本地文件、建立连接)
|
||||
:func:`MemoryProvider.system_prompt_block` 注入系统提示词的静态说明文本
|
||||
:func:`MemoryProvider.prefetch` 每轮对话前召回上下文
|
||||
:func:`MemoryProvider.queue_prefetch` 下一轮的后台预热
|
||||
:func:`MemoryProvider.sync_turn` 每轮对话后异步写入后端
|
||||
:func:`MemoryProvider.get_tool_schemas` 暴露给 LLM 的工具 schema
|
||||
:func:`MemoryProvider.handle_tool_call` 路由工具调用
|
||||
:func:`MemoryProvider.shutdown` 清理 (flush 队列、关连接)
|
||||
|
||||
可选 hooks
|
||||
==========
|
||||
仅按 ``生产推荐档`` 移植两个 ——
|
||||
- :func:`MemoryProvider.on_session_switch` thread_id 切换时 (branch/resume/reset/压缩后)
|
||||
- :func:`MemoryProvider.on_memory_write` builtin 写入时,通知外部 provider 镜像
|
||||
|
||||
参见 ``docs/MEMORY_V2_DESIGN_ZH.md`` §4 的完整接口定义和 §16 的实施分期。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryProvider(ABC):
|
||||
"""所有记忆 provider 必须实现的接口。"""
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""短标识,如 ``'builtin'`` / ``'hindsight'``。"""
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def is_available(self) -> bool:
|
||||
"""是否已配置并可用。
|
||||
|
||||
在 agent 启动时被调用,用于判断要不要激活该 provider。
|
||||
实现里只检查配置和依赖,不发网络请求。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""初始化:打开连接、创建 bank、加载本地文件等。
|
||||
|
||||
kwargs 至少包含 ``user_id`` / ``base_dir``,其余字段视 provider 而定。
|
||||
Provider 可以根据 ``agent_id is None`` 决定走全局还是 per-agent 路径。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
user_id:
|
||||
当前用户的隔离标识(无认证场景下为 ``"default"``)。
|
||||
agent_id:
|
||||
当前 agent 的稳定标识;``None`` 表示走全局/默认 agent。
|
||||
thread_id:
|
||||
当前对话的 thread id;某些 provider 用它做并发隔离或 bank 后缀。
|
||||
base_dir:
|
||||
``Paths.base_dir`` 字符串形式,避免每个 provider 重复解析路径。
|
||||
"""
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""清理:刷队列、关连接。默认 no-op。"""
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""注入系统提示词的静态文本。
|
||||
|
||||
典型用途:USER.md / MEMORY.md 的 frozen snapshot,provider 状态说明等。
|
||||
返回空串表示不贡献内容。
|
||||
"""
|
||||
return ""
|
||||
|
||||
# ---- 召回 / 写入 -----------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""对话前召回上下文。
|
||||
|
||||
在每次 LLM 调用前由 :class:`MemoryMiddleware` 触发。``query`` 通常
|
||||
是用户最近一条消息。实现要快;真正的网络召回应该走后台线程,
|
||||
这里直接返回缓存结果。
|
||||
|
||||
返回值:带格式的文本块,会被拼接到系统提示词末尾。空串表示无内容。
|
||||
"""
|
||||
return ""
|
||||
|
||||
def queue_prefetch(self, query: str, *, thread_id: str = "") -> None:
|
||||
"""后台预热下一轮的召回。默认 no-op。"""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""每轮对话结束后把对话持久化到后端。
|
||||
|
||||
实现应当非阻塞 —— 如果后端有延迟,排队到后台处理而不是直接调用。
|
||||
默认 no-op。
|
||||
|
||||
``turn_index`` 是当前对话的轮次序号 (从 1 开始,由编排层按对话历史算出)。
|
||||
provider 通常被每轮重新构造、无法靠自身计数,需要按轮次节流写入
|
||||
(例如 "每 N 轮 retain 一次") 的实现应当用它而非进程内计数器。
|
||||
缺省 ``None`` 表示调用方未提供。
|
||||
"""
|
||||
|
||||
# ---- LLM 工具 --------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""暴露给 LLM 的工具 schema 列表。
|
||||
|
||||
每个 schema 遵循 OpenAI function calling 格式::
|
||||
|
||||
{"name": "...", "description": "...", "parameters": {...}}
|
||||
|
||||
空数组表示 context-only(只做自动 recall/retain,不让 LLM 看见工具)。
|
||||
"""
|
||||
|
||||
def handle_tool_call(
|
||||
self, tool_name: str, args: dict[str, Any], **kwargs: Any
|
||||
) -> str:
|
||||
"""处理一个工具调用,必须返回 JSON 字符串。
|
||||
|
||||
``MemoryManager`` 会确保只把本 provider ``get_tool_schemas`` 里
|
||||
声明的工具名路由到这里。
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"Provider {self.name} does not handle tool {tool_name}"
|
||||
)
|
||||
|
||||
# ---- 可选 hooks (生产推荐档移植) -------------------------------------
|
||||
|
||||
def on_session_switch(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""thread_id 切换时触发。
|
||||
|
||||
deerflow 的 thread_id 在以下场景会变化:
|
||||
- 用户新建/切换 thread (``reset=True``)
|
||||
- LangGraph thread branch (``reset=False``,``parent_thread_id`` 为原 thread)
|
||||
- SummarizationMiddleware 压缩后继续 (``reset=False``)
|
||||
|
||||
Provider 在 ``initialize`` 时如果缓存了 per-session 的状态
|
||||
(``_session_turns`` 累积缓冲、bank 关联、查询缓存等),应当在
|
||||
这里刷新。默认 no-op。
|
||||
"""
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入时触发,通知外部 provider 镜像同一条记忆。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
action:
|
||||
``'add'`` / ``'replace'`` / ``'remove'``。
|
||||
target:
|
||||
``'user'`` / ``'memory'``,决定走全局 user bank 还是 agent 工作 bank。
|
||||
content:
|
||||
条目内容。``remove`` 时为待删除条目的内容(供 external 做幂等)。
|
||||
metadata:
|
||||
写入来源元数据 (``user_id`` / ``agent_id`` / ``thread_id`` / ``tool_name``)。
|
||||
|
||||
默认 no-op —— 只有需要镜像 builtin 写入的外部 provider (例如 Hindsight)
|
||||
才需要重写。``MemoryManager`` 调度时会跳过 builtin 自身,避免无意义的回环。
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["MemoryProvider"]
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"""Memory V2 - 具体 provider 实现。
|
||||
|
||||
- :class:`~deerflow.agents.memory.providers.builtin.BuiltinFileProvider`
|
||||
本地文件 provider (USER.md / MEMORY.md),始终启用。
|
||||
- ``HindsightProvider`` — Hindsight HTTP 客户端,P2 期接入。
|
||||
"""
|
||||
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
__all__ = ["BuiltinFileProvider"]
|
||||
+420
@@ -0,0 +1,420 @@
|
||||
"""Memory V2 - 本地文件 provider (USER.md / MEMORY.md)。
|
||||
|
||||
从 Hermes ``tools/memory_tool.py`` 的 ``MemoryStore`` 移植,适配 deerflow 的
|
||||
per-user / per-agent 目录结构与 :class:`MemoryProvider` 接口。
|
||||
|
||||
两类存储
|
||||
========
|
||||
- ``USER.md`` 用户画像 —— LLM 主动写入的 ``target="user"`` 记忆,跨 agent 共享。
|
||||
路径:``{base_dir}/users/{user_id}/USER.md``
|
||||
- ``MEMORY.md`` agent 工作笔记 —— LLM 主动写入的 ``target="memory"`` 记忆,agent 隔离。
|
||||
路径:``{base_dir}/users/{user_id}/agents/{agent_id}/MEMORY.md``
|
||||
|
||||
设计要点
|
||||
========
|
||||
- 条目以 ``\\n§\\n`` 分隔,支持多行;采用字符级配额 (与模型无关)。
|
||||
- frozen snapshot:``initialize`` 时读盘一次并冻结为系统提示词块,会话内写入
|
||||
直接落盘 (durable) 但不刷新快照 —— 保持系统提示词稳定,利于 prefix cache。
|
||||
- 读-改-写经文件锁串行化;写入用临时文件 + 原子 rename,避免并发读到半截文件。
|
||||
- 写入前经 :mod:`deerflow.agents.memory.security` 扫描注入/外渗。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.agents.memory.security import scan_memory_content
|
||||
from deerflow.config.memory_config import SecurityConfig
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 跨平台文件锁:Unix 用 fcntl,Windows 用 msvcrt。
|
||||
_fcntl: Any = None
|
||||
_msvcrt: Any = None
|
||||
try: # pragma: no cover - 平台相关
|
||||
import fcntl as _fcntl_mod
|
||||
|
||||
_fcntl = _fcntl_mod
|
||||
except ImportError: # pragma: no cover - Windows
|
||||
try:
|
||||
import msvcrt as _msvcrt_mod
|
||||
|
||||
_msvcrt = _msvcrt_mod
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
#: 条目分隔符。``§`` = section sign (U+00A7)。
|
||||
ENTRY_DELIMITER = "\n§\n"
|
||||
|
||||
#: target 取值。
|
||||
_VALID_TARGETS = ("memory", "user")
|
||||
|
||||
# 同一进程内对同一文件路径的写入串行化锁 (文件锁之上再加一层,降低自争用)。
|
||||
_PATH_LOCKS: dict[str, threading.Lock] = {}
|
||||
_PATH_LOCKS_GUARD = threading.Lock()
|
||||
|
||||
|
||||
def _get_path_lock(path: Path) -> threading.Lock:
|
||||
"""返回某文件路径对应的进程内锁 (按路径字符串缓存)。"""
|
||||
key = str(path)
|
||||
with _PATH_LOCKS_GUARD:
|
||||
lock = _PATH_LOCKS.get(key)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
_PATH_LOCKS[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _file_lock(path: Path):
|
||||
"""获取某文件的排他锁,保证读-改-写安全。
|
||||
|
||||
用单独的 ``.lock`` 文件加锁,这样记忆文件本身仍可被 ``os.replace`` 原子替换。
|
||||
进程内先抢 :func:`_get_path_lock`,再抢跨进程文件锁。
|
||||
"""
|
||||
process_lock = _get_path_lock(path)
|
||||
process_lock.acquire()
|
||||
lock_path = path.with_suffix(path.suffix + ".lock")
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if _fcntl is None and _msvcrt is None: # pragma: no cover - 无锁平台
|
||||
yield
|
||||
return
|
||||
if _msvcrt is not None and (not lock_path.exists() or lock_path.stat().st_size == 0):
|
||||
lock_path.write_text(" ", encoding="utf-8")
|
||||
fd = open(lock_path, "r+" if _msvcrt is not None else "a+")
|
||||
try:
|
||||
if _fcntl is not None:
|
||||
_fcntl.flock(fd, _fcntl.LOCK_EX)
|
||||
else: # pragma: no cover - Windows
|
||||
fd.seek(0)
|
||||
_msvcrt.locking(fd.fileno(), _msvcrt.LK_LOCK, 1)
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
if _fcntl is not None:
|
||||
_fcntl.flock(fd, _fcntl.LOCK_UN)
|
||||
elif _msvcrt is not None: # pragma: no cover - Windows
|
||||
fd.seek(0)
|
||||
_msvcrt.locking(fd.fileno(), _msvcrt.LK_UNLCK, 1)
|
||||
except OSError:
|
||||
pass
|
||||
fd.close()
|
||||
finally:
|
||||
process_lock.release()
|
||||
|
||||
|
||||
def _read_entries(path: Path) -> list[str]:
|
||||
"""读取一个记忆文件并按分隔符切成条目列表。
|
||||
|
||||
无需加锁:写入用原子 rename,读者总能看到旧文件或新文件的完整内容。
|
||||
"""
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return []
|
||||
if not raw.strip():
|
||||
return []
|
||||
entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)]
|
||||
return [e for e in entries if e]
|
||||
|
||||
|
||||
def _write_entries(path: Path, entries: list[str]) -> None:
|
||||
"""用临时文件 + 原子 rename 写入记忆文件。"""
|
||||
content = ENTRY_DELIMITER.join(entries) if entries else ""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".mem_")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class BuiltinFileProvider(MemoryProvider):
|
||||
"""本地文件记忆 provider,始终可用,不可移除。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
memory_char_limit: int = 2200,
|
||||
user_char_limit: int = 1375,
|
||||
deduplicate_on_load: bool = True,
|
||||
security: SecurityConfig | None = None,
|
||||
) -> None:
|
||||
self._memory_char_limit = memory_char_limit
|
||||
self._user_char_limit = user_char_limit
|
||||
self._deduplicate_on_load = deduplicate_on_load
|
||||
self._security = security or SecurityConfig()
|
||||
|
||||
self._user_id: str = DEFAULT_USER_ID
|
||||
self._agent_id: str = "default"
|
||||
self._user_path: Path | None = None
|
||||
self._memory_path: Path | None = None
|
||||
|
||||
# 实时状态 (工具写入后立即更新并落盘)
|
||||
self._user_entries: list[str] = []
|
||||
self._memory_entries: list[str] = []
|
||||
# frozen snapshot (initialize 时冻结,系统提示词注入用)
|
||||
self._snapshot: dict[str, str] = {"user": "", "memory": ""}
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "builtin"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""本地文件 provider 无外部依赖,始终可用。"""
|
||||
return True
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str = "",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""解析 per-user / per-agent 路径,加载文件并冻结系统提示词快照。"""
|
||||
self._user_id = user_id or DEFAULT_USER_ID
|
||||
self._agent_id = (agent_id or "default").lower()
|
||||
paths = get_paths()
|
||||
self._user_path = paths.user_profile_md_file(self._user_id)
|
||||
self._memory_path = paths.user_agent_memory_md_file(self._user_id, self._agent_id)
|
||||
self._load_from_disk()
|
||||
|
||||
def _load_from_disk(self) -> None:
|
||||
"""从磁盘加载两个文件,去重,并冻结系统提示词快照。"""
|
||||
assert self._user_path is not None and self._memory_path is not None
|
||||
self._user_entries = _read_entries(self._user_path)
|
||||
self._memory_entries = _read_entries(self._memory_path)
|
||||
if self._deduplicate_on_load:
|
||||
self._user_entries = list(dict.fromkeys(self._user_entries))
|
||||
self._memory_entries = list(dict.fromkeys(self._memory_entries))
|
||||
self._snapshot = {
|
||||
"user": self._render_block("user", self._user_entries),
|
||||
"memory": self._render_block("memory", self._memory_entries),
|
||||
}
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""返回 frozen snapshot:USER PROFILE 块 + MEMORY 块。
|
||||
|
||||
返回 ``initialize`` 时冻结的内容,会话内的工具写入不影响它 ——
|
||||
保持系统提示词稳定,利于 prefix cache。
|
||||
"""
|
||||
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
|
||||
content = "\n\n".join(blocks)
|
||||
if not content:
|
||||
return ""
|
||||
return f"<memory>\n{content}\n</memory>"
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
|
||||
|
||||
不经 MemoryManager 路由,因此这里返回空列表。
|
||||
"""
|
||||
return []
|
||||
|
||||
# ---- 工具写入操作 (供 memory 工具直接调用) ---------------------------
|
||||
|
||||
def add(self, target: str, content: str) -> dict[str, Any]:
|
||||
"""追加一条记忆条目。超出字符配额时返回错误。"""
|
||||
content = (content or "").strip()
|
||||
if not content:
|
||||
return {"success": False, "error": "条目内容不能为空。"}
|
||||
if (err := self._scan(content)) is not None:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
if content in entries:
|
||||
return self._success(target, "条目已存在,未重复添加。")
|
||||
limit = self._char_limit(target)
|
||||
new_total = len(ENTRY_DELIMITER.join([*entries, content]))
|
||||
if new_total > limit:
|
||||
current = self._char_count(entries)
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"记忆已用 {current:,}/{limit:,} 字符,"
|
||||
f"再加这条 ({len(content)} 字符) 会超出配额。"
|
||||
"请先 replace 或 remove 现有条目。"
|
||||
),
|
||||
"entries": entries,
|
||||
}
|
||||
entries.append(content)
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已添加。")
|
||||
|
||||
def replace(self, target: str, old_text: str, new_content: str) -> dict[str, Any]:
|
||||
"""用 ``old_text`` 短串定位条目,替换为 ``new_content``。"""
|
||||
old_text = (old_text or "").strip()
|
||||
new_content = (new_content or "").strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text 不能为空。"}
|
||||
if not new_content:
|
||||
return {"success": False, "error": "new_content 不能为空;删除条目请用 remove。"}
|
||||
if (err := self._scan(new_content)) is not None:
|
||||
return {"success": False, "error": err}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
if not matches:
|
||||
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
|
||||
if len({e for _, e in matches}) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
|
||||
"matches": previews,
|
||||
}
|
||||
idx = matches[0][0]
|
||||
limit = self._char_limit(target)
|
||||
test_entries = list(entries)
|
||||
test_entries[idx] = new_content
|
||||
new_total = len(ENTRY_DELIMITER.join(test_entries))
|
||||
if new_total > limit:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"替换后会达到 {new_total:,}/{limit:,} 字符,超出配额。"
|
||||
"请缩短新内容或先删除其他条目。"
|
||||
),
|
||||
}
|
||||
entries[idx] = new_content
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已替换。")
|
||||
|
||||
def remove(self, target: str, old_text: str) -> dict[str, Any]:
|
||||
"""用 ``old_text`` 短串定位并删除条目。"""
|
||||
old_text = (old_text or "").strip()
|
||||
if not old_text:
|
||||
return {"success": False, "error": "old_text 不能为空。"}
|
||||
|
||||
path = self._path_for(target)
|
||||
with _file_lock(path):
|
||||
entries = self._reload_target(target)
|
||||
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
|
||||
if not matches:
|
||||
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
|
||||
if len({e for _, e in matches}) > 1:
|
||||
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
|
||||
"matches": previews,
|
||||
}
|
||||
entries.pop(matches[0][0])
|
||||
_write_entries(path, entries)
|
||||
self._set_entries(target, entries)
|
||||
return self._success(target, "条目已删除。")
|
||||
|
||||
def read(self, target: str) -> list[str]:
|
||||
"""返回某 target 的实时条目列表 (从磁盘重读)。"""
|
||||
return self._reload_target(target)
|
||||
|
||||
def usage(self, target: str) -> str:
|
||||
"""返回某 target 的字符用量描述,如 ``'12% — 165/1375 字符'``。"""
|
||||
entries = self._reload_target(target)
|
||||
current = self._char_count(entries)
|
||||
limit = self._char_limit(target)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
return f"{pct}% — {current:,}/{limit:,} 字符"
|
||||
|
||||
# ---- 内部辅助 --------------------------------------------------------
|
||||
|
||||
def _scan(self, content: str) -> str | None:
|
||||
return scan_memory_content(
|
||||
content,
|
||||
scan_threats=self._security.scan_content,
|
||||
block_invisible_unicode=self._security.block_invisible_unicode,
|
||||
)
|
||||
|
||||
def _path_for(self, target: str) -> Path:
|
||||
if target not in _VALID_TARGETS:
|
||||
raise ValueError(f"非法 target '{target}',只能是 'memory' 或 'user'。")
|
||||
if self._user_path is None or self._memory_path is None:
|
||||
raise RuntimeError("BuiltinFileProvider 未初始化,请先调用 initialize()。")
|
||||
return self._user_path if target == "user" else self._memory_path
|
||||
|
||||
def _char_limit(self, target: str) -> int:
|
||||
return self._user_char_limit if target == "user" else self._memory_char_limit
|
||||
|
||||
@staticmethod
|
||||
def _char_count(entries: list[str]) -> int:
|
||||
return len(ENTRY_DELIMITER.join(entries)) if entries else 0
|
||||
|
||||
def _reload_target(self, target: str) -> list[str]:
|
||||
"""加锁状态下从磁盘重读条目 (拿到其他会话写入的最新状态)。"""
|
||||
entries = _read_entries(self._path_for(target))
|
||||
if self._deduplicate_on_load:
|
||||
entries = list(dict.fromkeys(entries))
|
||||
self._set_entries(target, entries)
|
||||
return entries
|
||||
|
||||
def _set_entries(self, target: str, entries: list[str]) -> None:
|
||||
if target == "user":
|
||||
self._user_entries = entries
|
||||
else:
|
||||
self._memory_entries = entries
|
||||
|
||||
def _success(self, target: str, message: str) -> dict[str, Any]:
|
||||
entries = self._user_entries if target == "user" else self._memory_entries
|
||||
current = self._char_count(entries)
|
||||
limit = self._char_limit(target)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
return {
|
||||
"success": True,
|
||||
"target": target,
|
||||
"entries": entries,
|
||||
"entry_count": len(entries),
|
||||
"usage": f"{pct}% — {current:,}/{limit:,} 字符",
|
||||
"message": message,
|
||||
}
|
||||
|
||||
def _render_block(self, target: str, entries: list[str]) -> str:
|
||||
"""渲染一个系统提示词块 (带表头和用量指示)。"""
|
||||
if not entries:
|
||||
return ""
|
||||
limit = self._char_limit(target)
|
||||
content = ENTRY_DELIMITER.join(entries)
|
||||
current = len(content)
|
||||
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
|
||||
if target == "user":
|
||||
header = f"用户画像 USER PROFILE [{pct}% — {current:,}/{limit:,} 字符]"
|
||||
else:
|
||||
header = f"记忆笔记 MEMORY [{pct}% — {current:,}/{limit:,} 字符]"
|
||||
separator = "═" * 46
|
||||
return f"{separator}\n{header}\n{separator}\n{content}"
|
||||
|
||||
|
||||
__all__ = ["BuiltinFileProvider", "ENTRY_DELIMITER"]
|
||||
+462
@@ -0,0 +1,462 @@
|
||||
"""Memory V2 - Hindsight 外部 provider。
|
||||
|
||||
Hindsight 是带知识图谱、实体消解、多策略检索的长期记忆服务。本 provider 以
|
||||
HTTP 客户端方式接入一个用户自起的 Hindsight Docker 实例 (``local_external`` 模式)。
|
||||
|
||||
双 bank 隔离 (B 方案,详见 docs/MEMORY_V2_DESIGN_ZH.md §3)
|
||||
==========================================================
|
||||
- 全局 user bank ``bank_id_template`` 例:``deerflow-user-{user_id}``
|
||||
存用户身份、偏好,跨 agent 共享。
|
||||
- agent 工作 bank ``work_bank_id_template`` 例:``deerflow-work-{user_id}-{agent_id}``
|
||||
存 agent 私有工作笔记。
|
||||
|
||||
写入路由:
|
||||
- LLM 主动 ``memory(target="user")`` -> 镜像到 user bank
|
||||
- LLM 主动 ``memory(target="memory")`` -> 镜像到 work bank
|
||||
- 自动 retain 的对话 (``sync_turn``) -> work bank
|
||||
|
||||
召回:``prefetch`` 并发查两个 bank,合并结果。
|
||||
|
||||
document_id 与轮次计数
|
||||
======================
|
||||
本 provider 被 ``build_memory_manager`` 每轮重新构造 —— 是无状态的。因此:
|
||||
- ``document_id`` 每次 ``initialize`` 用带时间戳的新值,每轮对话的 retain 落到
|
||||
各自独立的 Hindsight 文档。
|
||||
- ``retain_every_n_turns`` 的轮次判定依赖外部传入的 ``turn_index``
|
||||
(由 ``MemoryMiddleware`` 按对话历史算出);未传时退回进程内 ``_turn_counter``。
|
||||
|
||||
客户端生命周期
|
||||
==============
|
||||
``_run_coro`` 每次操作新建一个事件循环。aiohttp ``ClientSession`` 是
|
||||
loop-bound 的,跨 loop 复用会泄漏并失效。因此本 provider **不缓存客户端** ——
|
||||
每次记忆操作在 :meth:`_run_with_client` 里即开即关,保证 session 在其所属的
|
||||
loop 内被正常关闭。
|
||||
|
||||
降级
|
||||
====
|
||||
``hindsight-client`` 是可选依赖。未安装或未配置 ``api_url`` 时 :meth:`is_available`
|
||||
返回 ``False``,provider 不会被注册,记忆系统退回纯 builtin —— 不影响任何功能。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
from deerflow.config.memory_config import HindsightMemoryConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: hindsight-client 最低版本。
|
||||
MIN_CLIENT_VERSION = "0.4.22"
|
||||
|
||||
|
||||
def hindsight_client_available() -> bool:
|
||||
"""返回 ``hindsight-client`` 包是否已安装 (不导入实际客户端)。"""
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec("hindsight_client") is not None
|
||||
|
||||
|
||||
def _run_coro(coro: Any, timeout: float) -> Any:
|
||||
"""把一个 async coro 跑到完成并返回结果 (带超时)。
|
||||
|
||||
deerflow 的中间件钩子有同步也有异步路径;本 helper 两种场景都能用:
|
||||
- 当前线程没有运行中的事件循环 -> 直接 ``asyncio.run``
|
||||
- 当前线程已在事件循环里 -> 另起线程跑,避免嵌套循环报错
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if not in_loop:
|
||||
return asyncio.run(asyncio.wait_for(coro, timeout))
|
||||
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def _runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(asyncio.wait_for(coro, timeout))
|
||||
except BaseException as exc: # noqa: BLE001 - 跨线程传递异常
|
||||
box["error"] = exc
|
||||
|
||||
thread = threading.Thread(target=_runner, name="hindsight-coro", daemon=True)
|
||||
thread.start()
|
||||
thread.join(timeout + 5)
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
if "value" not in box:
|
||||
raise TimeoutError("Hindsight 操作超时")
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class HindsightProvider(MemoryProvider):
|
||||
"""Hindsight 外部记忆 provider。"""
|
||||
|
||||
def __init__(self, config: HindsightMemoryConfig | None = None) -> None:
|
||||
self._config = config or HindsightMemoryConfig()
|
||||
|
||||
self._user_id: str = ""
|
||||
self._agent_id: str = "default"
|
||||
self._thread_id: str = ""
|
||||
self._user_bank: str = ""
|
||||
self._work_bank: str = ""
|
||||
|
||||
# retain 批次的 document id 与轮次计数 (on_session_switch 会重置)
|
||||
self._document_id: str = ""
|
||||
self._turn_counter: int = 0
|
||||
|
||||
# ---- 身份 ------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "hindsight"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""配置完整且 hindsight-client 已安装时才可用。"""
|
||||
if not hindsight_client_available():
|
||||
logger.debug("Hindsight 不可用:hindsight-client 未安装")
|
||||
return False
|
||||
if not self._config.api_url:
|
||||
logger.debug("Hindsight 不可用:未配置 api_url")
|
||||
return False
|
||||
return True
|
||||
|
||||
# ---- 生命周期 ---------------------------------------------------------
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str = "",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""解析双 bank 名称,准备 retain 批次状态。客户端每次操作即开即关。"""
|
||||
self._user_id = user_id or "default"
|
||||
self._agent_id = (agent_id or "default").lower()
|
||||
self._thread_id = thread_id or ""
|
||||
self._user_bank = self._resolve_bank(self._config.bank_id_template)
|
||||
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
|
||||
self._reset_retain_batch()
|
||||
|
||||
def _resolve_bank(self, template: str) -> str:
|
||||
"""用 user_id / agent_id / thread_id 填充 bank 名称模板。
|
||||
|
||||
空占位符会塌缩 (例如 ``deerflow-{thread_id}`` 在无 thread 时变成 ``deerflow-``,
|
||||
再清理多余连字符)。
|
||||
"""
|
||||
name = template.format(
|
||||
user_id=self._user_id,
|
||||
agent_id=self._agent_id,
|
||||
thread_id=self._thread_id,
|
||||
)
|
||||
# 清理空占位符塌缩留下的多余连字符
|
||||
while "--" in name:
|
||||
name = name.replace("--", "-")
|
||||
return name.strip("-") or "deerflow"
|
||||
|
||||
def _reset_retain_batch(self) -> None:
|
||||
"""重置 retain 批次状态 (新的 document_id + 轮次清零)。
|
||||
|
||||
每次 ``initialize`` 用带时间戳的新 ``document_id`` —— 每轮对话的 retain
|
||||
落到各自独立的 Hindsight 文档。轮次计数清零,仅作为 ``turn_index`` 缺失
|
||||
时的退路。
|
||||
"""
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
self._document_id = f"{self._thread_id or 'thread'}-{ts}"
|
||||
self._turn_counter = 0
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""无需清理 —— 客户端在每次操作内即开即关,不缓存。"""
|
||||
|
||||
# ---- 客户端生命周期 --------------------------------------------------
|
||||
|
||||
def _client_factory(self) -> Any:
|
||||
"""创建一个新的 Hindsight HTTP 客户端。
|
||||
|
||||
每次记忆操作都新建一个、用完即关,**不缓存** —— aiohttp ClientSession
|
||||
是 loop-bound 的,而 ``_run_coro`` 每次新建事件循环。测试可覆盖本方法
|
||||
注入 fake 客户端。
|
||||
"""
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"base_url": self._config.api_url,
|
||||
"timeout": 30.0,
|
||||
}
|
||||
if self._config.api_key:
|
||||
kwargs["api_key"] = self._config.api_key
|
||||
return Hindsight(**kwargs)
|
||||
|
||||
@staticmethod
|
||||
async def _aclose_client(client: Any) -> None:
|
||||
"""关闭客户端 (兼容暴露 ``aclose`` 或 ``close`` 的实现)。"""
|
||||
close = getattr(client, "aclose", None) or getattr(client, "close", None)
|
||||
if close is None:
|
||||
return
|
||||
try:
|
||||
result = close()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception as exc: # noqa: BLE001 - 关闭失败不影响结果
|
||||
logger.debug("Hindsight 客户端关闭失败:%s", exc)
|
||||
|
||||
async def _run_with_client(self, op: Any) -> Any:
|
||||
"""在当前事件循环内创建客户端、执行 ``op(client)``、保证关闭客户端。
|
||||
|
||||
``op`` 是一个接收 client、返回 awaitable 的协程函数。无论成功、异常还是
|
||||
被取消 (超时),``finally`` 都会关闭客户端 —— 杜绝 ``Unclosed client session``。
|
||||
"""
|
||||
client = self._client_factory()
|
||||
try:
|
||||
return await op(client)
|
||||
finally:
|
||||
await self._aclose_client(client)
|
||||
|
||||
# ---- 系统提示词块 -----------------------------------------------------
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
"""返回一小段静态说明 (告诉 LLM Hindsight 处于哪种模式)。"""
|
||||
mode = self._config.memory_mode
|
||||
if mode == "context":
|
||||
return "" # context 模式纯后台,无需告知 LLM
|
||||
if mode == "tools":
|
||||
return (
|
||||
"# Hindsight 长期记忆\n"
|
||||
"可用 hindsight_recall 检索、hindsight_reflect 综合、hindsight_retain 存储。"
|
||||
)
|
||||
return (
|
||||
"# Hindsight 长期记忆\n"
|
||||
"相关记忆会自动注入上下文;也可用 hindsight_recall / hindsight_reflect / "
|
||||
"hindsight_retain 主动检索与存储。"
|
||||
)
|
||||
|
||||
# ---- 召回 ------------------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
"""并发召回两个 bank 并合并结果。
|
||||
|
||||
``memory_mode='tools'`` 或 ``auto_recall=False`` 时不做自动召回。
|
||||
召回失败 (Hindsight 不可达等) 返回空串,不抛异常。
|
||||
"""
|
||||
if self._config.memory_mode == "tools":
|
||||
return ""
|
||||
if not self._config.auto_recall:
|
||||
return ""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return ""
|
||||
if self._config.recall_max_input_chars and len(query) > self._config.recall_max_input_chars:
|
||||
query = query[: self._config.recall_max_input_chars]
|
||||
|
||||
try:
|
||||
merged = self._recall_both(query)
|
||||
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
|
||||
logger.debug("Hindsight 召回失败 (非致命):%s", exc)
|
||||
return ""
|
||||
|
||||
if not merged:
|
||||
return ""
|
||||
lines = "\n".join(f"- {text}" for text in merged)
|
||||
return (
|
||||
"# Hindsight 记忆召回 (跨会话持久上下文)\n"
|
||||
"下面是与当前对话相关的历史记忆,可据此回答关于用户和过往会话的问题。\n\n"
|
||||
f"{lines}"
|
||||
)
|
||||
|
||||
def _recall_both(self, query: str) -> list[str]:
|
||||
"""并发召回 user + work 两个 bank,合并去重。一次操作共用一个客户端。"""
|
||||
|
||||
async def _op(client: Any) -> tuple[Any, Any]:
|
||||
return await asyncio.gather(
|
||||
client.arecall(
|
||||
bank_id=self._user_bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
max_tokens=self._config.recall_max_tokens,
|
||||
),
|
||||
client.arecall(
|
||||
bank_id=self._work_bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
max_tokens=self._config.recall_max_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
user_resp, work_resp = _run_coro(self._run_with_client(_op), timeout=15)
|
||||
return self._merge_results(
|
||||
self._extract_results(user_resp),
|
||||
self._extract_results(work_resp),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_results(resp: Any) -> list[tuple[float, str]]:
|
||||
"""从一个 recall 响应里抽出 ``[(score, text), ...]``。"""
|
||||
results = getattr(resp, "results", None) or []
|
||||
out: list[tuple[float, str]] = []
|
||||
for r in results:
|
||||
text = (getattr(r, "text", "") or "").strip()
|
||||
if text:
|
||||
out.append((float(getattr(r, "score", 0.0) or 0.0), text))
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _merge_results(
|
||||
user_results: list[tuple[float, str]],
|
||||
work_results: list[tuple[float, str]],
|
||||
) -> list[str]:
|
||||
"""合并双 bank 召回结果:按 score 倒序,去重保序。"""
|
||||
combined = sorted(user_results + work_results, key=lambda x: -x[0])
|
||||
seen: set[str] = set()
|
||||
merged: list[str] = []
|
||||
for _, text in combined:
|
||||
if text not in seen:
|
||||
seen.add(text)
|
||||
merged.append(text)
|
||||
return merged
|
||||
|
||||
# ---- 写入 ------------------------------------------------------------
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""自动 retain 一轮对话到 work bank。
|
||||
|
||||
``auto_retain=False`` 时不写。按 ``retain_every_n_turns`` 控制频率。
|
||||
|
||||
轮次判定优先用调用方传入的 ``turn_index`` (由 MemoryMiddleware 按对话历史
|
||||
算出) —— 因为本 provider 每轮被重建、进程内 ``_turn_counter`` 会被重置,
|
||||
单靠它在 ``retain_every_n_turns > 1`` 时会永远不触发。``turn_index`` 缺失
|
||||
时才退回 ``_turn_counter``。
|
||||
"""
|
||||
if not self._config.auto_retain:
|
||||
return
|
||||
|
||||
if turn_index is not None:
|
||||
counter = turn_index
|
||||
else:
|
||||
self._turn_counter += 1
|
||||
counter = self._turn_counter
|
||||
|
||||
every = max(1, self._config.retain_every_n_turns)
|
||||
if counter % every != 0:
|
||||
return
|
||||
|
||||
turn_text = (
|
||||
f"{self._config.retain_user_prefix}: {user_content}\n"
|
||||
f"{self._config.retain_assistant_prefix}: {assistant_content}"
|
||||
)
|
||||
try:
|
||||
self._retain(self._work_bank, turn_text, kind="conversation")
|
||||
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
|
||||
logger.warning("Hindsight sync_turn 写入失败:%s", exc)
|
||||
|
||||
def on_memory_write(
|
||||
self,
|
||||
action: str,
|
||||
target: str,
|
||||
content: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""builtin 写入时镜像到 Hindsight。
|
||||
|
||||
``target="user"`` -> user bank;``target="memory"`` -> work bank。
|
||||
目前只镜像 ``add`` / ``replace`` 的新内容 (Hindsight 是 append-only 语义,
|
||||
``remove`` 不反向删除)。
|
||||
"""
|
||||
if action not in ("add", "replace"):
|
||||
return
|
||||
content = (content or "").strip()
|
||||
if not content:
|
||||
return
|
||||
bank = self._user_bank if target == "user" else self._work_bank
|
||||
try:
|
||||
self._retain(bank, content, kind=f"builtin_{target}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Hindsight on_memory_write 镜像失败:%s", exc)
|
||||
|
||||
def _retain(self, bank_id: str, content: str, *, kind: str) -> None:
|
||||
"""向某 bank 写入一条记忆。客户端即开即关。"""
|
||||
metadata = {
|
||||
"retained_at": _utc_now_iso(),
|
||||
"kind": kind,
|
||||
"user_id": self._user_id,
|
||||
"agent_id": self._agent_id,
|
||||
}
|
||||
if self._thread_id:
|
||||
metadata["thread_id"] = self._thread_id
|
||||
|
||||
async def _op(client: Any) -> Any:
|
||||
return await client.aretain(
|
||||
bank_id=bank_id,
|
||||
content=content,
|
||||
context=self._config.bank_retain_mission or None,
|
||||
metadata=metadata,
|
||||
document_id=self._document_id,
|
||||
retain_async=self._config.retain_async,
|
||||
)
|
||||
|
||||
_run_coro(self._run_with_client(_op), timeout=15)
|
||||
|
||||
def reflect(self, query: str, *, use_user_bank: bool = True) -> str:
|
||||
"""对某 bank 做跨记忆综合 (LLM 推理)。供 hindsight_reflect 工具调用。"""
|
||||
bank = self._user_bank if use_user_bank else self._work_bank
|
||||
|
||||
async def _op(client: Any) -> Any:
|
||||
return await client.areflect(
|
||||
bank_id=bank,
|
||||
query=query,
|
||||
budget=self._config.recall_budget,
|
||||
)
|
||||
|
||||
resp = _run_coro(self._run_with_client(_op), timeout=30)
|
||||
return getattr(resp, "text", "") or ""
|
||||
|
||||
def recall(self, query: str) -> list[str]:
|
||||
"""召回双 bank 并返回文本列表。供 hindsight_recall 工具调用。"""
|
||||
return self._recall_both(query)
|
||||
|
||||
def retain_to_user_bank(self, content: str) -> None:
|
||||
"""主动写入 user bank。供 hindsight_retain 工具调用。"""
|
||||
self._retain(self._user_bank, content, kind="tool_retain")
|
||||
|
||||
# ---- LLM 工具 --------------------------------------------------------
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""Hindsight 的 3 个工具以 LangChain BaseTool 独立注册,不经 manager 路由。"""
|
||||
return []
|
||||
|
||||
# ---- 可选 hooks ------------------------------------------------------
|
||||
|
||||
def on_session_switch(
|
||||
self,
|
||||
new_thread_id: str,
|
||||
*,
|
||||
parent_thread_id: str = "",
|
||||
reset: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""thread 切换:重新解析 work bank,重置 retain 批次。"""
|
||||
self._thread_id = new_thread_id
|
||||
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
|
||||
self._reset_retain_batch()
|
||||
|
||||
|
||||
__all__ = ["HindsightProvider", "hindsight_client_available", "MIN_CLIENT_VERSION"]
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"""Read-only provider for the mounted policy knowledge base.
|
||||
|
||||
This provider keeps the current knowledge-base behaviour: inject the lightweight
|
||||
``routing.md`` guide and let the model decide when to read concrete files. It
|
||||
does not automatically retrieve snippets and never writes to the knowledge base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _read_routing_file(host_path: str) -> str:
|
||||
path = Path(host_path)
|
||||
if not path.is_file():
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
class KnowledgeProvider(MemoryProvider):
|
||||
"""Injects the knowledge routing guide as a read-only context provider."""
|
||||
|
||||
def __init__(self, *, app_config: Any | None = None) -> None:
|
||||
self._app_config = app_config
|
||||
self._routing_host_path: Path | None = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "knowledge"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
app_config = self._app_config or kwargs.get("app_config")
|
||||
if app_config is None:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
app_config = get_app_config()
|
||||
|
||||
self._routing_host_path = None
|
||||
for mount in app_config.sandbox.mounts or []:
|
||||
if mount.container_path.rstrip("/") == "/mnt/knowledge":
|
||||
self._routing_host_path = Path(mount.host_path) / "routing.md"
|
||||
break
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
if self._routing_host_path is None:
|
||||
return ""
|
||||
|
||||
content = _read_routing_file(str(self._routing_host_path.resolve()))
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
logger.debug("Injected knowledge routing memory from %s", self._routing_host_path)
|
||||
return f"""<knowledge_routing_memory source="/mnt/knowledge/routing.md">
|
||||
{content}
|
||||
</knowledge_routing_memory>"""
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
__all__ = ["KnowledgeProvider"]
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based MemoryProvider 实现。
|
||||
|
||||
存储格式:``{base_dir}/users/{user_id}/memory.json``(经 FileMemoryStorage 管理)
|
||||
JSON 结构使用嵌套 ``{summary, updatedAt}`` 对象,与原版 DeerFlow 完全一致。
|
||||
|
||||
LLM 在 V1 模式下不调用 memory 工具 —— 事实完全由异步 LLM 提取。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _FakeMsg:
|
||||
"""轻量 message 对象,用于将文本传给原版 format_conversation_for_update。"""
|
||||
__slots__ = ("type", "content")
|
||||
|
||||
def __init__(self, type_: str, content: str) -> None:
|
||||
self.type = type_
|
||||
self.content = content
|
||||
|
||||
|
||||
class V1BuiltinProvider(MemoryProvider):
|
||||
"""原版 DeerFlow V1 记忆 provider。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_facts: int = 100,
|
||||
fact_confidence_threshold: float = 0.7,
|
||||
debounce_seconds: int = 30,
|
||||
model_name: str = "",
|
||||
token_budget: int = 2000,
|
||||
) -> None:
|
||||
self._max_facts = max_facts
|
||||
self._confidence_threshold = fact_confidence_threshold
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._model_name = model_name
|
||||
self._token_budget = token_budget
|
||||
|
||||
self._user_id: str = ""
|
||||
self._thread_id: str = ""
|
||||
self._agent_id: str | None = None
|
||||
self._frozen_block: str = ""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "v1_builtin"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
_ = base_dir # FileMemoryStorage uses get_paths() internally
|
||||
self._user_id = user_id
|
||||
self._thread_id = thread_id or ""
|
||||
self._agent_id = agent_id
|
||||
self._frozen_block = self._build_frozen_block()
|
||||
|
||||
# ---- 系统提示词 -------------------------------------------------------
|
||||
|
||||
def _build_frozen_block(self) -> str:
|
||||
"""从 memory.json 读取数据,用 format_memory_for_injection 渲染注入块。
|
||||
|
||||
当 agent_id 已知时加载 per-agent 记忆;否则加载用户级记忆。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.memory.v1.prompt import format_memory_for_injection
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
|
||||
data = get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
text = format_memory_for_injection(data, max_tokens=self._token_budget)
|
||||
if not text:
|
||||
return ""
|
||||
return f"<memory>\n{text}\n</memory>"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider 加载 memory.json 失败: %s", exc)
|
||||
return ""
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
return self._frozen_block
|
||||
|
||||
# ---- 召回 / 写入 ------------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
# V1 无动态召回,静态 snapshot 已在 system_prompt_block 里
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""把本轮对话加入防抖队列,等待 LLM 异步提取事实。"""
|
||||
if not user_content.strip() and not assistant_content.strip():
|
||||
return
|
||||
try:
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
|
||||
msgs = [
|
||||
_FakeMsg("human", user_content),
|
||||
_FakeMsg("ai", assistant_content),
|
||||
]
|
||||
queue = get_memory_queue()
|
||||
queue.add(
|
||||
thread_id=thread_id or self._thread_id,
|
||||
messages=msgs,
|
||||
user_id=self._user_id,
|
||||
agent_name=self._agent_id or None,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider sync_turn 入队失败: %s", exc)
|
||||
|
||||
# ---- 工具 (V1 不暴露工具给 LLM) -------------------------------------
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
# ---- API 读取接口 (供 Gateway router 用) -----------------------------
|
||||
|
||||
def read_v1_data(self) -> dict[str, Any]:
|
||||
"""读取 memory.json 的当前内容(实时读磁盘,非 frozen snapshot)。
|
||||
|
||||
当 agent_id 已知时读取 per-agent 记忆;否则读用户级记忆。
|
||||
"""
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
return get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
|
||||
def fact_count(self) -> int:
|
||||
try:
|
||||
return len(self.read_v1_data().get("facts", []))
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
|
||||
def usage_text(self) -> str:
|
||||
return f"{self.fact_count()}/{self._max_facts} 个事实"
|
||||
|
||||
|
||||
__all__ = ["V1BuiltinProvider"]
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Memory V2 - 记忆内容安全扫描。
|
||||
|
||||
记忆条目会被注入到系统提示词里,因此写入前必须扫描,阻断:
|
||||
- Prompt 注入 / 角色劫持 / 隐藏指令
|
||||
- 凭证外渗 (curl/wget 带密钥、读 .env 等敏感文件)
|
||||
- 持久化后门 (写 authorized_keys 等)
|
||||
- 不可见 unicode (零宽字符、双向控制符)
|
||||
|
||||
这是从 Hermes ``tools/memory_tool.py`` 的 ``_scan_memory_content`` 移植而来,
|
||||
并针对 deerflow 的目录结构调整了敏感路径模式。
|
||||
|
||||
扫描默认开启,可由 ``config.memory.security.scan_content`` /
|
||||
``block_invisible_unicode`` 关闭(不建议)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 威胁正则模式表:(正则, 模式 id)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_THREAT_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
# --- Prompt 注入 / 角色劫持 ---
|
||||
# 允许多个限定词,如 "ignore ALL above instructions"
|
||||
(re.compile(r"ignore\s+(?:(?:previous|all|above|prior|any)\s+)+instructions", re.IGNORECASE), "prompt_injection"),
|
||||
(re.compile(r"忽略(?:以上|之前|上述|所有|全部|一切)+(?:的)?(?:指令|指示|要求|提示词)"), "prompt_injection_zh"),
|
||||
(re.compile(r"you\s+are\s+now\s+", re.IGNORECASE), "role_hijack"),
|
||||
(re.compile(r"do\s+not\s+tell\s+the\s+user", re.IGNORECASE), "deception_hide"),
|
||||
(re.compile(r"不要(告诉|让)用户(知道)?"), "deception_hide_zh"),
|
||||
(re.compile(r"system\s+prompt\s+override", re.IGNORECASE), "sys_prompt_override"),
|
||||
(re.compile(r"disregard\s+(your|all|any)\s+(instructions|rules|guidelines)", re.IGNORECASE), "disregard_rules"),
|
||||
(re.compile(r"act\s+as\s+(if|though)\s+you\s+(have\s+no|don'?t\s+have)\s+(restrictions|limits|rules)", re.IGNORECASE), "bypass_restrictions"),
|
||||
# --- 凭证外渗 ---
|
||||
(re.compile(r"curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_curl"),
|
||||
(re.compile(r"wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_wget"),
|
||||
(re.compile(r"cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)", re.IGNORECASE), "read_secrets"),
|
||||
# --- 持久化后门 ---
|
||||
(re.compile(r"authorized_keys", re.IGNORECASE), "ssh_backdoor"),
|
||||
(re.compile(r"(\$HOME|~)/\.ssh", re.IGNORECASE), "ssh_access"),
|
||||
]
|
||||
|
||||
# 不可见 unicode 字符:零宽字符 + 双向控制符。
|
||||
# 它们肉眼不可见,常被用来在记忆里藏注入 payload。
|
||||
_INVISIBLE_CHARS: frozenset[str] = frozenset(
|
||||
{
|
||||
"", # ZERO WIDTH SPACE
|
||||
"", # ZERO WIDTH NON-JOINER
|
||||
"", # ZERO WIDTH JOINER
|
||||
"", # WORD JOINER
|
||||
"", # ZERO WIDTH NO-BREAK SPACE / BOM
|
||||
"", # LEFT-TO-RIGHT EMBEDDING
|
||||
"", # RIGHT-TO-LEFT EMBEDDING
|
||||
"", # POP DIRECTIONAL FORMATTING
|
||||
"", # LEFT-TO-RIGHT OVERRIDE
|
||||
"", # RIGHT-TO-LEFT OVERRIDE
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class MemoryContentBlocked(ValueError):
|
||||
"""记忆内容被安全扫描阻断时抛出。
|
||||
|
||||
``pattern_id`` 标明命中的威胁模式,便于上层返回结构化错误。
|
||||
"""
|
||||
|
||||
def __init__(self, pattern_id: str, message: str) -> None:
|
||||
self.pattern_id = pattern_id
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def scan_memory_content(
|
||||
content: str,
|
||||
*,
|
||||
scan_threats: bool = True,
|
||||
block_invisible_unicode: bool = True,
|
||||
) -> str | None:
|
||||
"""扫描记忆内容,命中威胁时返回错误描述,干净则返回 ``None``。
|
||||
|
||||
不抛异常 —— 返回字符串方便工具层直接塞进 ``{"success": False, "error": ...}``。
|
||||
需要异常语义时用 :func:`assert_memory_content_safe`。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
content:
|
||||
待写入的记忆条目内容。
|
||||
scan_threats:
|
||||
是否扫描注入/外渗模式。对应 ``config.memory.security.scan_content``。
|
||||
block_invisible_unicode:
|
||||
是否阻断不可见 unicode。对应 ``config.memory.security.block_invisible_unicode``。
|
||||
"""
|
||||
if block_invisible_unicode:
|
||||
for char in _INVISIBLE_CHARS:
|
||||
if char in content:
|
||||
return (
|
||||
f"已阻断:内容包含不可见 unicode 字符 U+{ord(char):04X},"
|
||||
"可能是注入 payload。"
|
||||
)
|
||||
|
||||
if scan_threats:
|
||||
for pattern, pattern_id in _THREAT_PATTERNS:
|
||||
if pattern.search(content):
|
||||
return (
|
||||
f"已阻断:内容命中威胁模式 '{pattern_id}'。"
|
||||
"记忆条目会被注入系统提示词,不允许包含注入或外渗内容。"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def assert_memory_content_safe(
|
||||
content: str,
|
||||
*,
|
||||
scan_threats: bool = True,
|
||||
block_invisible_unicode: bool = True,
|
||||
) -> None:
|
||||
"""扫描记忆内容,命中威胁时抛出 :class:`MemoryContentBlocked`。
|
||||
|
||||
:func:`scan_memory_content` 的异常版本,供需要异常语义的调用方使用。
|
||||
"""
|
||||
error = scan_memory_content(
|
||||
content,
|
||||
scan_threats=scan_threats,
|
||||
block_invisible_unicode=block_invisible_unicode,
|
||||
)
|
||||
if error is not None:
|
||||
# 从错误描述里反推 pattern_id 不可靠,这里重新扫一遍拿确切 id。
|
||||
pattern_id = "invisible_unicode"
|
||||
for pattern, pid in _THREAT_PATTERNS:
|
||||
if pattern.search(content):
|
||||
pattern_id = pid
|
||||
break
|
||||
raise MemoryContentBlocked(pattern_id, error)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryContentBlocked",
|
||||
"scan_memory_content",
|
||||
"assert_memory_content_safe",
|
||||
]
|
||||
+1
@@ -0,0 +1 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based 记忆子模块。"""
|
||||
+371
@@ -0,0 +1,371 @@
|
||||
"""Prompt templates for memory update and injection."""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
TIKTOKEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
TIKTOKEN_AVAILABLE = False
|
||||
|
||||
# Prompt template for updating memory based on conversation
|
||||
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
|
||||
|
||||
Current Memory State:
|
||||
<current_memory>
|
||||
{current_memory}
|
||||
</current_memory>
|
||||
|
||||
New Conversation to Process:
|
||||
<conversation>
|
||||
{conversation}
|
||||
</conversation>
|
||||
|
||||
Instructions:
|
||||
1. Analyze the conversation for important information about the user
|
||||
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
|
||||
3. Update the memory sections as needed following the detailed length guidelines below
|
||||
|
||||
Before extracting facts, perform a structured reflection on the conversation:
|
||||
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
|
||||
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
|
||||
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
|
||||
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
|
||||
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
|
||||
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
|
||||
If yes, record them as facts with the most appropriate category and confidence.
|
||||
|
||||
{correction_hint}
|
||||
|
||||
Memory Section Guidelines:
|
||||
|
||||
**User Context** (Current state - concise summaries):
|
||||
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
|
||||
Example: Core contributor, project names with metrics (16k+ stars), technical stack
|
||||
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
|
||||
Example: Bilingual capabilities, specific interest areas, expertise domains
|
||||
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
|
||||
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
|
||||
Include: Active implementation work, troubleshooting issues, market/research interests
|
||||
Note: This captures SEVERAL concurrent focus areas, not just one task
|
||||
|
||||
**History** (Temporal context - rich paragraphs; updated incrementally each conversation):
|
||||
- recentMonths: Running log of recent work and activities (4-6 sentences or 1-2 paragraphs)
|
||||
Scope: Accumulate from each conversation — any task the user worked on, problems solved, technologies touched
|
||||
Update policy: Set shouldUpdate=true whenever the conversation contains any task, discussion, or work activity; integrate new content into existing text rather than overwriting
|
||||
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
|
||||
- earlierContext: Important patterns promoted from recentMonths over time (3-5 sentences or 1 paragraph)
|
||||
Scope: Recurring themes and established patterns across multiple conversations
|
||||
Update policy: Set shouldUpdate=true when a pattern repeats or an older item in recentMonths should be archived here
|
||||
Include: Past projects, learning journeys, established patterns
|
||||
- longTermBackground: Persistent foundational context that rarely changes (2-4 sentences)
|
||||
Scope: Core expertise, fundamental working style, long-standing interests
|
||||
Update policy: Set shouldUpdate=true only when a genuinely new foundational fact emerges
|
||||
Include: Core expertise, longstanding interests, fundamental working style
|
||||
|
||||
**Facts Extraction**:
|
||||
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
|
||||
- Include proper nouns (company names, project names, technology names)
|
||||
- Preserve technical terminology and version numbers
|
||||
- Categories:
|
||||
* preference: Tools, styles, approaches user prefers/dislikes
|
||||
* knowledge: Specific expertise, technologies mastered, domain knowledge
|
||||
* context: Background facts (job title, projects, locations, languages)
|
||||
* behavior: Working patterns, communication habits, problem-solving approaches
|
||||
* goal: Stated objectives, learning targets, project ambitions
|
||||
* correction: Explicit agent mistakes or user corrections, including the correct approach
|
||||
- Confidence levels:
|
||||
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
|
||||
* 0.7-0.8: Strongly implied from actions/discussions
|
||||
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
|
||||
|
||||
**What Goes Where**:
|
||||
- workContext: Current job, active projects, primary tech stack
|
||||
- personalContext: Languages, personality, interests outside direct work tasks
|
||||
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
|
||||
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
|
||||
- recentMonths: Cumulative log of recent work — update from EVERY conversation with meaningful content
|
||||
- earlierContext: Recurring patterns promoted from recentMonths; update when patterns solidify
|
||||
- longTermBackground: Unchanging foundational facts about the user; update only for new foundational info
|
||||
|
||||
**Language Requirement**:
|
||||
- ALL summaries and fact content MUST be written in Chinese (Simplified)
|
||||
- Keep technical terms, proper nouns, product names, and company names in their original form (e.g., DeepSeek, LangGraph, GitHub)
|
||||
- Surrounding descriptive text must be in Chinese regardless of the conversation language
|
||||
|
||||
Output Format (JSON):
|
||||
{{
|
||||
"user": {{
|
||||
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"history": {{
|
||||
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"newFacts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
],
|
||||
"factsToRemove": ["fact_id_1", "fact_id_2"]
|
||||
}}
|
||||
|
||||
Important Rules:
|
||||
- For recentMonths: set shouldUpdate=true whenever the conversation contains any task, work, or substantive discussion — this field accumulates like a log
|
||||
- For earlierContext / longTermBackground: set shouldUpdate=true only when a new pattern solidifies or a foundational fact changes
|
||||
- For user context fields: set shouldUpdate=true if there's meaningful new information
|
||||
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
|
||||
- Include specific metrics, version numbers, and proper nouns in facts
|
||||
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
|
||||
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
|
||||
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
|
||||
- Remove facts that are contradicted by new information
|
||||
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
|
||||
Keep 3-5 concurrent focus themes that are still active and relevant
|
||||
- For history sections, integrate new information chronologically into appropriate time period
|
||||
- Preserve technical accuracy - keep exact names of technologies, companies, projects
|
||||
- Focus on information useful for future interactions and personalization
|
||||
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
|
||||
session-specific and ephemeral — they will not be accessible in future sessions.
|
||||
Recording upload events causes confusion in subsequent conversations.
|
||||
- CRITICAL: Write ALL summary text and fact content in Chinese (Simplified). Technical terms and proper nouns (product names, company names, technology names) may remain in their original language, but all other text must be in Chinese.
|
||||
|
||||
Return ONLY valid JSON, no explanation or markdown."""
|
||||
|
||||
|
||||
# Prompt template for extracting facts from a single message
|
||||
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
|
||||
|
||||
Message:
|
||||
{message}
|
||||
|
||||
Extract facts in this JSON format:
|
||||
{{
|
||||
"facts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
]
|
||||
}}
|
||||
|
||||
Categories:
|
||||
- preference: User preferences (likes/dislikes, styles, tools)
|
||||
- knowledge: User's expertise or knowledge areas
|
||||
- context: Background context (location, job, projects)
|
||||
- behavior: Behavioral patterns
|
||||
- goal: User's goals or objectives
|
||||
- correction: Explicit corrections or mistakes to avoid repeating
|
||||
|
||||
Rules:
|
||||
- Only extract clear, specific facts
|
||||
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
|
||||
- Skip vague or temporary information
|
||||
- Write all fact content in Chinese (Simplified); technical terms and proper nouns may remain in their original language
|
||||
|
||||
Return ONLY valid JSON."""
|
||||
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
|
||||
"""Count tokens in text using tiktoken.
|
||||
|
||||
Args:
|
||||
text: The text to count tokens for.
|
||||
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
|
||||
|
||||
Returns:
|
||||
The number of tokens in the text.
|
||||
"""
|
||||
if not TIKTOKEN_AVAILABLE:
|
||||
return len(text) // 4
|
||||
|
||||
try:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
return len(encoding.encode(text))
|
||||
except Exception:
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
|
||||
"""Coerce a confidence-like value to a bounded float in [0, 1].
|
||||
|
||||
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
|
||||
to the default before clamping, preventing them from dominating ranking.
|
||||
The ``default`` parameter is assumed to be a finite value.
|
||||
"""
|
||||
try:
|
||||
confidence = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return max(0.0, min(1.0, default))
|
||||
if not math.isfinite(confidence):
|
||||
return max(0.0, min(1.0, default))
|
||||
return max(0.0, min(1.0, confidence))
|
||||
|
||||
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
|
||||
"""Format memory data for injection into system prompt.
|
||||
|
||||
Args:
|
||||
memory_data: The memory data dictionary.
|
||||
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
|
||||
|
||||
Returns:
|
||||
Formatted memory string for system prompt injection.
|
||||
"""
|
||||
if not memory_data:
|
||||
return ""
|
||||
|
||||
sections = []
|
||||
|
||||
# Format user context
|
||||
user_data = memory_data.get("user", {})
|
||||
if user_data:
|
||||
user_sections = []
|
||||
|
||||
work_ctx = user_data.get("workContext", {})
|
||||
if work_ctx.get("summary"):
|
||||
user_sections.append(f"Work: {work_ctx['summary']}")
|
||||
|
||||
personal_ctx = user_data.get("personalContext", {})
|
||||
if personal_ctx.get("summary"):
|
||||
user_sections.append(f"Personal: {personal_ctx['summary']}")
|
||||
|
||||
top_of_mind = user_data.get("topOfMind", {})
|
||||
if top_of_mind.get("summary"):
|
||||
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
|
||||
|
||||
if user_sections:
|
||||
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
|
||||
|
||||
# Format history
|
||||
history_data = memory_data.get("history", {})
|
||||
if history_data:
|
||||
history_sections = []
|
||||
|
||||
recent = history_data.get("recentMonths", {})
|
||||
if recent.get("summary"):
|
||||
history_sections.append(f"Recent: {recent['summary']}")
|
||||
|
||||
earlier = history_data.get("earlierContext", {})
|
||||
if earlier.get("summary"):
|
||||
history_sections.append(f"Earlier: {earlier['summary']}")
|
||||
|
||||
background = history_data.get("longTermBackground", {})
|
||||
if background.get("summary"):
|
||||
history_sections.append(f"Background: {background['summary']}")
|
||||
|
||||
if history_sections:
|
||||
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
|
||||
|
||||
# Format facts (sorted by confidence; include as many as token budget allows)
|
||||
facts_data = memory_data.get("facts", [])
|
||||
if isinstance(facts_data, list) and facts_data:
|
||||
ranked_facts = sorted(
|
||||
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
|
||||
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
base_text = "\n\n".join(sections)
|
||||
base_tokens = _count_tokens(base_text) if base_text else 0
|
||||
facts_header = "Facts:\n"
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
|
||||
running_tokens = base_tokens + separator_tokens
|
||||
|
||||
fact_lines: list[str] = []
|
||||
for fact in ranked_facts:
|
||||
content_value = fact.get("content")
|
||||
if not isinstance(content_value, str):
|
||||
continue
|
||||
content = content_value.strip()
|
||||
if not content:
|
||||
continue
|
||||
category = str(fact.get("category", "context")).strip() or "context"
|
||||
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
|
||||
source_error = fact.get("sourceError")
|
||||
if category == "correction" and isinstance(source_error, str) and source_error.strip():
|
||||
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
|
||||
else:
|
||||
line = f"- [{category} | {confidence:.2f}] {content}"
|
||||
|
||||
line_text = ("\n" + line) if fact_lines else line
|
||||
line_tokens = _count_tokens(line_text)
|
||||
|
||||
if running_tokens + line_tokens <= max_tokens:
|
||||
fact_lines.append(line)
|
||||
running_tokens += line_tokens
|
||||
else:
|
||||
break
|
||||
|
||||
if fact_lines:
|
||||
sections.append("Facts:\n" + "\n".join(fact_lines))
|
||||
|
||||
if not sections:
|
||||
return ""
|
||||
|
||||
result = "\n\n".join(sections)
|
||||
|
||||
token_count = _count_tokens(result)
|
||||
if token_count > max_tokens:
|
||||
char_per_token = len(result) / token_count
|
||||
target_chars = int(max_tokens * char_per_token * 0.95)
|
||||
result = result[:target_chars] + "\n..."
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_conversation_for_update(messages: list[Any]) -> str:
|
||||
"""Format conversation messages for memory update prompt.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
|
||||
Returns:
|
||||
Formatted conversation string.
|
||||
"""
|
||||
lines = []
|
||||
for msg in messages:
|
||||
role = getattr(msg, "type", "unknown")
|
||||
content = getattr(msg, "content", str(msg))
|
||||
|
||||
# Handle content that might be a list (multimodal)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for p in content:
|
||||
if isinstance(p, str):
|
||||
text_parts.append(p)
|
||||
elif isinstance(p, dict):
|
||||
text_val = p.get("text")
|
||||
if isinstance(text_val, str):
|
||||
text_parts.append(text_val)
|
||||
content = " ".join(text_parts) if text_parts else str(content)
|
||||
|
||||
# Strip uploaded_files tags from human messages to avoid persisting
|
||||
# ephemeral file path info into long-term memory. Skip the turn entirely
|
||||
# when nothing remains after stripping (upload-only message).
|
||||
if role == "human":
|
||||
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Truncate very long messages
|
||||
if len(str(content)) > 1000:
|
||||
content = str(content)[:1000] + "..."
|
||||
|
||||
if role == "human":
|
||||
lines.append(f"User: {content}")
|
||||
elif role == "ai":
|
||||
lines.append(f"Assistant: {content}")
|
||||
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MEMORY_UPDATE_PROMPT",
|
||||
"FACT_EXTRACTION_PROMPT",
|
||||
"format_memory_for_injection",
|
||||
"format_conversation_for_update",
|
||||
"_count_tokens",
|
||||
"_coerce_confidence",
|
||||
]
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Memory V1 — 防抖异步更新队列(原版 DeerFlow MemoryUpdateQueue)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
def _memory_enabled() -> bool:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().enabled
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationContext:
|
||||
"""Context for a conversation to be processed for memory update."""
|
||||
thread_id: str
|
||||
messages: list[Any]
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
agent_name: str | None = None
|
||||
user_id: str | None = None
|
||||
correction_detected: bool = False
|
||||
reinforcement_detected: bool = False
|
||||
|
||||
|
||||
class MemoryUpdateQueue:
|
||||
"""Queue for memory updates with debounce mechanism.
|
||||
|
||||
Collects conversation contexts and processes them after a configurable
|
||||
debounce period. Multiple conversations received within the debounce
|
||||
window are deduplicated by (thread_id, user_id, agent_name).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue: list[ConversationContext] = []
|
||||
self._lock = threading.Lock()
|
||||
self._timer: threading.Timer | None = None
|
||||
self._processing = False
|
||||
|
||||
@staticmethod
|
||||
def _queue_key(thread_id: str, user_id: str | None, agent_name: str | None) -> tuple[str, str | None, str | None]:
|
||||
return (thread_id, user_id, agent_name)
|
||||
|
||||
def add(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation to the update queue (with debounce)."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._reset_timer()
|
||||
|
||||
logger.info("V1 memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def add_nowait(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation and start processing immediately in the background."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._schedule_timer(0)
|
||||
|
||||
logger.info("V1 memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def _enqueue_locked(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
user_id: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
) -> None:
|
||||
queue_key = self._queue_key(thread_id, user_id, agent_name)
|
||||
|
||||
existing = next(
|
||||
(ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) == queue_key),
|
||||
None,
|
||||
)
|
||||
|
||||
merged_correction = correction_detected or (existing.correction_detected if existing is not None else False)
|
||||
merged_reinforcement = reinforcement_detected or (existing.reinforcement_detected if existing is not None else False)
|
||||
|
||||
context = ConversationContext(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=merged_correction,
|
||||
reinforcement_detected=merged_reinforcement,
|
||||
)
|
||||
|
||||
self._queue = [ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) != queue_key]
|
||||
self._queue.append(context)
|
||||
|
||||
def _reset_timer(self) -> None:
|
||||
cfg = _get_v1_cfg()
|
||||
self._schedule_timer(cfg.debounce_seconds)
|
||||
logger.debug("V1 memory update timer set for %ss", cfg.debounce_seconds)
|
||||
|
||||
def _schedule_timer(self, delay_seconds: float) -> None:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = threading.Timer(delay_seconds, self._process_queue)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _process_queue(self) -> None:
|
||||
from deerflow.agents.memory.v1.updater import MemoryUpdater
|
||||
|
||||
with self._lock:
|
||||
if self._processing:
|
||||
self._schedule_timer(0)
|
||||
return
|
||||
if not self._queue:
|
||||
return
|
||||
self._processing = True
|
||||
contexts_to_process = self._queue.copy()
|
||||
self._queue.clear()
|
||||
self._timer = None
|
||||
|
||||
logger.info("V1 memory: processing %d queued updates", len(contexts_to_process))
|
||||
|
||||
try:
|
||||
updater = MemoryUpdater()
|
||||
for context in contexts_to_process:
|
||||
try:
|
||||
logger.info("V1 memory: updating for thread %s", context.thread_id)
|
||||
success = updater.update_memory(
|
||||
messages=context.messages,
|
||||
thread_id=context.thread_id,
|
||||
agent_name=context.agent_name,
|
||||
correction_detected=context.correction_detected,
|
||||
reinforcement_detected=context.reinforcement_detected,
|
||||
user_id=context.user_id,
|
||||
)
|
||||
if success:
|
||||
logger.info("V1 memory: updated successfully for thread %s", context.thread_id)
|
||||
else:
|
||||
logger.warning("V1 memory: update skipped/failed for thread %s", context.thread_id)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("V1 memory: error updating for thread %s: %s", context.thread_id, e)
|
||||
|
||||
if len(contexts_to_process) > 1:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processing = False
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force immediate processing of the queue."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._process_queue()
|
||||
|
||||
def flush_nowait(self) -> None:
|
||||
"""Start queue processing immediately in a background thread."""
|
||||
with self._lock:
|
||||
self._schedule_timer(0)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear the queue without processing (for testing)."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._queue.clear()
|
||||
self._processing = False
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._queue)
|
||||
|
||||
@property
|
||||
def is_processing(self) -> bool:
|
||||
with self._lock:
|
||||
return self._processing
|
||||
|
||||
|
||||
_memory_queue: MemoryUpdateQueue | None = None
|
||||
_queue_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_queue() -> MemoryUpdateQueue:
|
||||
"""Get the global memory update queue singleton."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is None:
|
||||
_memory_queue = MemoryUpdateQueue()
|
||||
return _memory_queue
|
||||
|
||||
|
||||
def reset_memory_queue() -> None:
|
||||
"""Reset the global memory queue (for testing)."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is not None:
|
||||
_memory_queue.clear()
|
||||
_memory_queue = None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConversationContext",
|
||||
"MemoryUpdateQueue",
|
||||
"get_memory_queue",
|
||||
"reset_memory_queue",
|
||||
]
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
"""Memory V1 — 存储层(原版 DeerFlow 格式)。
|
||||
|
||||
JSON 结构:
|
||||
{
|
||||
"version": "1.0",
|
||||
"lastUpdated": "...",
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground":{"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"facts": []
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def utc_now_iso_z() -> str:
|
||||
"""Current UTC time as ISO-8601 with ``Z`` suffix."""
|
||||
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
|
||||
|
||||
|
||||
def create_empty_memory() -> dict[str, Any]:
|
||||
"""Create an empty memory structure (new format)."""
|
||||
return {
|
||||
"version": "1.0",
|
||||
"lastUpdated": utc_now_iso_z(),
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"facts": [],
|
||||
}
|
||||
|
||||
|
||||
def _migrate_legacy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把老格式 {userContext: {workContext: str}} 迁移到新格式 {user: {workContext: {summary, updatedAt}}}。"""
|
||||
if "userContext" not in data:
|
||||
return data
|
||||
|
||||
now = utc_now_iso_z()
|
||||
old_ctx = data.pop("userContext", {})
|
||||
data["user"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in old_ctx.items()
|
||||
}
|
||||
|
||||
# history 字段可能是纯字符串,需要包裹
|
||||
if "history" in data:
|
||||
hist = data["history"]
|
||||
data["history"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in hist.items()
|
||||
}
|
||||
|
||||
# 确保 user 包含所有必要字段
|
||||
base = create_empty_memory()
|
||||
for key in ("user", "history"):
|
||||
section = base[key]
|
||||
for field in section:
|
||||
if field not in data.get(key, {}):
|
||||
data.setdefault(key, {})[field] = section[field]
|
||||
|
||||
data["version"] = "1.0"
|
||||
data.pop("updatedAt", None)
|
||||
return data
|
||||
|
||||
|
||||
class FileMemoryStorage:
|
||||
"""File-based memory storage provider (原版 DeerFlow 实现)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Per-(user_id, agent_name) cache: value = (memory_data, file_mtime)
|
||||
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
def _validate_agent_name(self, agent_name: str) -> None:
|
||||
if not agent_name:
|
||||
raise ValueError("Agent name must be a non-empty string.")
|
||||
if not AGENT_NAME_PATTERN.match(agent_name):
|
||||
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
|
||||
|
||||
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
|
||||
paths = get_paths()
|
||||
if user_id is not None:
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.user_agent_memory_file(user_id, agent_name)
|
||||
return paths.user_memory_file(user_id)
|
||||
# Legacy: no user_id
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.agent_memory_file(agent_name)
|
||||
return paths.memory_file
|
||||
|
||||
def _load_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
if not file_path.exists():
|
||||
return create_empty_memory()
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return create_empty_memory()
|
||||
data = _migrate_legacy(data)
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning("Failed to load memory file %s: %s", file_path, e)
|
||||
return create_empty_memory()
|
||||
|
||||
@staticmethod
|
||||
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
|
||||
return (user_id, agent_name)
|
||||
|
||||
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data (cached with file modification time check)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
current_mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
cached = self._memory_cache.get(cache_key)
|
||||
if cached is not None and cached[1] == current_mtime:
|
||||
return cached[0]
|
||||
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, current_mtime)
|
||||
|
||||
return memory_data
|
||||
|
||||
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Reload memory data from file, forcing cache invalidation."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
return memory_data
|
||||
|
||||
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Save memory data to file and update cache (atomic write)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
|
||||
|
||||
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(memory_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
temp_path.replace(file_path)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
logger.info("Memory saved to %s", file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error("Failed to save memory file: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
_storage_instance: FileMemoryStorage | None = None
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_storage() -> FileMemoryStorage:
|
||||
"""Get the global FileMemoryStorage singleton."""
|
||||
global _storage_instance
|
||||
if _storage_instance is not None:
|
||||
return _storage_instance
|
||||
with _storage_lock:
|
||||
if _storage_instance is None:
|
||||
_storage_instance = FileMemoryStorage()
|
||||
return _storage_instance
|
||||
|
||||
|
||||
__all__ = [
|
||||
"create_empty_memory",
|
||||
"utc_now_iso_z",
|
||||
"FileMemoryStorage",
|
||||
"get_memory_storage",
|
||||
]
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""Hooks fired before summarization removes messages from state(原版 DeerFlow)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
|
||||
|
||||
def memory_flush_hook(event: object) -> None:
|
||||
"""Flush messages about to be summarized into the V1 memory queue.
|
||||
|
||||
``event`` is expected to be a ``SummarizationEvent``-like object with:
|
||||
- ``thread_id: str``
|
||||
- ``messages_to_summarize: list``
|
||||
- ``agent_name: str | None``
|
||||
- ``runtime: RunnableConfig``
|
||||
"""
|
||||
if not get_memory_config().enabled:
|
||||
return
|
||||
|
||||
thread_id = getattr(event, "thread_id", None)
|
||||
if not thread_id:
|
||||
return
|
||||
|
||||
messages_to_summarize = getattr(event, "messages_to_summarize", [])
|
||||
filtered_messages = filter_messages_for_memory(list(messages_to_summarize))
|
||||
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
|
||||
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
|
||||
if not user_messages or not assistant_messages:
|
||||
return
|
||||
|
||||
correction_detected = detect_correction(filtered_messages)
|
||||
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
|
||||
|
||||
try:
|
||||
from deerflow.runtime.user_context import resolve_runtime_user_id
|
||||
runtime = getattr(event, "runtime", None)
|
||||
user_id = resolve_runtime_user_id(runtime) if runtime is not None else None
|
||||
except Exception: # noqa: BLE001
|
||||
user_id = None
|
||||
|
||||
agent_name = getattr(event, "agent_name", None)
|
||||
queue = get_memory_queue()
|
||||
queue.add_nowait(
|
||||
thread_id=thread_id,
|
||||
messages=filtered_messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["memory_flush_hook"]
|
||||
+495
@@ -0,0 +1,495 @@
|
||||
"""Memory V1 — updater(原版 DeerFlow MemoryUpdater)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.v1.prompt import (
|
||||
MEMORY_UPDATE_PROMPT,
|
||||
format_conversation_for_update,
|
||||
)
|
||||
from deerflow.agents.memory.v1.storage import (
|
||||
create_empty_memory,
|
||||
get_memory_storage,
|
||||
utc_now_iso_z,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=4,
|
||||
thread_name_prefix="v1-memory-updater-sync",
|
||||
)
|
||||
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
"""返回 V1MemoryConfig(含 max_facts, fact_confidence_threshold, model_name)。"""
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 向后兼容的便捷函数(供 gateway router 等调用)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().reload(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
storage = get_memory_storage()
|
||||
if not storage.save(memory_data, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save imported memory data")
|
||||
return storage.load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
cleared = create_empty_memory()
|
||||
if not get_memory_storage().save(cleared, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save cleared memory data")
|
||||
return cleared
|
||||
|
||||
|
||||
def _validate_confidence(confidence: float) -> float:
|
||||
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
|
||||
raise ValueError("confidence")
|
||||
return confidence
|
||||
|
||||
|
||||
def create_memory_fact(
|
||||
content: str,
|
||||
category: str = "context",
|
||||
confidence: float = 0.5,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_content = content.strip()
|
||||
if not normalized_content:
|
||||
raise ValueError("content")
|
||||
normalized_category = category.strip() or "context"
|
||||
validated_confidence = _validate_confidence(confidence)
|
||||
now = utc_now_iso_z()
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
facts = list(memory_data.get("facts", []))
|
||||
facts.append({
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": normalized_category,
|
||||
"confidence": validated_confidence,
|
||||
"createdAt": now,
|
||||
"source": "manual",
|
||||
})
|
||||
updated_memory["facts"] = facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save memory data after creating fact")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
facts = memory_data.get("facts", [])
|
||||
updated_facts = [f for f in facts if f.get("id") != fact_id]
|
||||
if len(updated_facts) == len(facts):
|
||||
raise KeyError(fact_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def update_memory_fact(
|
||||
fact_id: str,
|
||||
content: str | None = None,
|
||||
category: str | None = None,
|
||||
confidence: float | None = None,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_facts: list[dict[str, Any]] = []
|
||||
found = False
|
||||
for fact in memory_data.get("facts", []):
|
||||
if fact.get("id") == fact_id:
|
||||
found = True
|
||||
updated_fact = dict(fact)
|
||||
if content is not None:
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
raise ValueError("content")
|
||||
updated_fact["content"] = normalized
|
||||
if category is not None:
|
||||
updated_fact["category"] = category.strip() or "context"
|
||||
if confidence is not None:
|
||||
updated_fact["confidence"] = _validate_confidence(confidence)
|
||||
updated_facts.append(updated_fact)
|
||||
else:
|
||||
updated_facts.append(fact)
|
||||
if not found:
|
||||
raise KeyError(fact_id)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内容提取辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
"""从 LLM response content(str 或 list of blocks)提取纯文本。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
pieces: list[str] = []
|
||||
pending: list[str] = []
|
||||
|
||||
def flush():
|
||||
if pending:
|
||||
pieces.append("".join(pending))
|
||||
pending.clear()
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
pending.append(block)
|
||||
elif isinstance(block, dict):
|
||||
flush()
|
||||
text_val = block.get("text")
|
||||
if isinstance(text_val, str):
|
||||
pieces.append(text_val)
|
||||
flush()
|
||||
return "\n".join(pieces)
|
||||
return str(content)
|
||||
|
||||
|
||||
_UPLOAD_SENTENCE_RE = re.compile(
|
||||
r"[^.!?]*\b(?:"
|
||||
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
|
||||
r"|file\s+upload"
|
||||
r"|/mnt/user-data/uploads/"
|
||||
r"|<uploaded_files>"
|
||||
r")[^.!?]*[.!?]?\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""移除记忆中关于文件上传事件的句子(上传文件是会话级临时的)。"""
|
||||
for section in ("user", "history"):
|
||||
section_data = memory_data.get(section, {})
|
||||
for _key, val in section_data.items():
|
||||
if isinstance(val, dict) and "summary" in val:
|
||||
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
|
||||
cleaned = re.sub(r" +", " ", cleaned)
|
||||
val["summary"] = cleaned
|
||||
|
||||
facts = memory_data.get("facts", [])
|
||||
if facts:
|
||||
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
|
||||
return memory_data
|
||||
|
||||
|
||||
def _fact_content_key(content: Any) -> str | None:
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
stripped = content.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
return stripped.casefold()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryUpdater(原版核心类)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class MemoryUpdater:
|
||||
"""Updates memory using LLM based on conversation context."""
|
||||
|
||||
def __init__(self, model_name: str | None = None) -> None:
|
||||
self._model_name = model_name
|
||||
|
||||
def _get_model(self):
|
||||
from deerflow.models import create_chat_model
|
||||
cfg = _get_v1_cfg()
|
||||
model_name = self._model_name or cfg.model_name
|
||||
return create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
|
||||
def _build_correction_hint(self, correction_detected: bool, reinforcement_detected: bool) -> str:
|
||||
correction_hint = ""
|
||||
if correction_detected:
|
||||
correction_hint = (
|
||||
"IMPORTANT: Explicit correction signals were detected in this conversation. "
|
||||
"Pay special attention to what the agent got wrong, what the user corrected, "
|
||||
"and record the correct approach as a fact with category "
|
||||
'"correction" and confidence >= 0.95 when appropriate.'
|
||||
)
|
||||
if reinforcement_detected:
|
||||
reinforcement_hint = (
|
||||
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
|
||||
"The user explicitly confirmed the agent's approach was correct or helpful. "
|
||||
"Record the confirmed approach, style, or preference as a fact with category "
|
||||
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
|
||||
)
|
||||
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
|
||||
return correction_hint
|
||||
|
||||
def _prepare_update_prompt(
|
||||
self,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
user_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], str] | None:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
config = get_memory_config()
|
||||
if not config.enabled or not messages:
|
||||
return None
|
||||
|
||||
current_memory = get_memory_data(agent_name, user_id=user_id)
|
||||
conversation_text = format_conversation_for_update(messages)
|
||||
if not conversation_text.strip():
|
||||
return None
|
||||
|
||||
correction_hint = self._build_correction_hint(correction_detected, reinforcement_detected)
|
||||
prompt = MEMORY_UPDATE_PROMPT.format(
|
||||
current_memory=json.dumps(current_memory, indent=2, ensure_ascii=False),
|
||||
conversation=conversation_text,
|
||||
correction_hint=correction_hint,
|
||||
)
|
||||
return current_memory, prompt
|
||||
|
||||
def _finalize_update(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
response_content: Any,
|
||||
thread_id: str | None,
|
||||
agent_name: str | None,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
response_text = _extract_text(response_content).strip()
|
||||
if response_text.startswith("```"):
|
||||
lines = response_text.split("\n")
|
||||
response_text = "\n".join(lines[1:-1] if lines[-1] == "```" else lines[1:])
|
||||
|
||||
update_data = json.loads(response_text)
|
||||
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
|
||||
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
|
||||
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
|
||||
|
||||
async def aupdate_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
return await asyncio.to_thread(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _do_update_memory_sync(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
prepared = self._prepare_update_prompt(
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
if prepared is None:
|
||||
return False
|
||||
|
||||
current_memory, prompt = prepared
|
||||
model = self._get_model()
|
||||
response = model.invoke(prompt, config={"run_name": "v1_memory_agent"})
|
||||
return self._finalize_update(
|
||||
current_memory=current_memory,
|
||||
response_content=response.content,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("V1 memory: failed to parse LLM response: %s", e)
|
||||
return False
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("V1 memory update failed: %s", e)
|
||||
return False
|
||||
|
||||
def update_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop is not None and loop.is_running():
|
||||
try:
|
||||
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
return future.result()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("V1 memory: failed to offload update to executor")
|
||||
return False
|
||||
|
||||
return self._do_update_memory_sync(
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _apply_updates(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
update_data: dict[str, Any],
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = _get_v1_cfg()
|
||||
now = utc_now_iso_z()
|
||||
|
||||
# 更新 user context 段
|
||||
user_updates = update_data.get("user", {})
|
||||
for section in ("workContext", "personalContext", "topOfMind"):
|
||||
section_data = user_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("user", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 更新 history 段
|
||||
history_updates = update_data.get("history", {})
|
||||
for section in ("recentMonths", "earlierContext", "longTermBackground"):
|
||||
section_data = history_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("history", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 删除事实
|
||||
facts_to_remove = set(update_data.get("factsToRemove", []))
|
||||
if facts_to_remove:
|
||||
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
|
||||
|
||||
# 新增事实(去重 + 置信度过滤)
|
||||
existing_keys = {
|
||||
key for key in (_fact_content_key(f.get("content")) for f in current_memory.get("facts", []))
|
||||
if key is not None
|
||||
}
|
||||
|
||||
for fact in update_data.get("newFacts", []):
|
||||
confidence = fact.get("confidence", 0.5)
|
||||
if confidence < cfg.fact_confidence_threshold:
|
||||
continue
|
||||
raw_content = fact.get("content", "")
|
||||
if not isinstance(raw_content, str):
|
||||
continue
|
||||
normalized_content = raw_content.strip()
|
||||
fact_key = _fact_content_key(normalized_content)
|
||||
if fact_key is not None and fact_key in existing_keys:
|
||||
continue
|
||||
|
||||
fact_entry: dict[str, Any] = {
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": fact.get("category", "context"),
|
||||
"confidence": confidence,
|
||||
"createdAt": now,
|
||||
"source": thread_id or "unknown",
|
||||
}
|
||||
source_error = fact.get("sourceError")
|
||||
if isinstance(source_error, str) and source_error.strip():
|
||||
fact_entry["sourceError"] = source_error.strip()
|
||||
|
||||
current_memory["facts"].append(fact_entry)
|
||||
if fact_key is not None:
|
||||
existing_keys.add(fact_key)
|
||||
|
||||
# 超出上限时驱逐置信度最低的事实
|
||||
if len(current_memory["facts"]) > cfg.max_facts:
|
||||
current_memory["facts"] = sorted(
|
||||
current_memory["facts"],
|
||||
key=lambda f: f.get("confidence", 0),
|
||||
reverse=True,
|
||||
)[:cfg.max_facts]
|
||||
|
||||
return current_memory
|
||||
|
||||
|
||||
def update_memory_from_conversation(
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
updater = MemoryUpdater()
|
||||
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryUpdater",
|
||||
"update_memory_from_conversation",
|
||||
"get_memory_data",
|
||||
"reload_memory_data",
|
||||
"import_memory_data",
|
||||
"clear_memory_data",
|
||||
"create_memory_fact",
|
||||
"delete_memory_fact",
|
||||
"update_memory_fact",
|
||||
]
|
||||
Reference in new issue
Block a user