feat(memory): Memory V2 重写 - 可插拔 provider + Hindsight 集成

以 Hermes 的可插拔记忆架构为骨架重写 deerflow 记忆模块,兼容原有
结构化数据,并新增 Hindsight 作为外部记忆后端。

P1 - 抽象层 + builtin 本地记忆:
- MemoryProvider 抽象基类 + MemoryManager 编排层(builtin + ≤1 外部)
- BuiltinFileProvider:USER.md/MEMORY.md 二元存储、字符配额、文件锁、
  原子写、frozen snapshot
- memory 工具(LangChain @tool):LLM 主动写入用户画像/工作笔记
- 记忆内容安全扫描(注入/外渗模式 + 不可见 unicode)
- memory_config 改为嵌套结构 + 旧扁平字段兼容 + per-user 三层覆盖
- 迁移脚本 migrate_memory_to_v2.py:旧 memory.json -> USER.md/MEMORY.md
- Gateway 新增 /api/memory/v2 与 v2/config 端点(用户可配置白名单字段)

P2 - Hindsight 外部 provider:
- HindsightProvider:双 bank 隔离(user bank 跨 agent 共享 + work bank
  agent 私有)、并发召回合并、auto-retain、on_memory_write 镜像
- hindsight_recall/reflect/retain 三个 LangChain 工具
- MemoryMiddleware.before_model 注入 Hindsight 动态召回
- context/tools/hybrid 三种 memory_mode
- hindsight-client 作为可选依赖,未装时自动退回纯 builtin

注入走系统提示词(builtin frozen snapshot)+ before_model(Hindsight
动态召回)。后台抽取保留但默认关闭,P3 期重接入。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
xuhdandClaude Opus 4.7 committed 2026-05-15 12:15:55 +08:00
1 parent 6947a3ffa7
commit f9d7610679
21 files changed
+3970 -188

No files matched your search

@@ -1,23 +1,38 @@
"""Memory module for DeerFlow.
"""DeerFlow 记忆模块 (Memory V2)。
This module provides a global memory mechanism that:
- Stores user context and conversation history in memory.json
- Uses LLM to summarize and extract facts from conversations
- Injects relevant memory into system prompts for personalized responses
V2 架构 (详见 docs/MEMORY_V2_DESIGN_ZH.md):
- :class:`MemoryProvider` 可插拔记忆 provider 抽象基类
- :class:`MemoryManager` 编排内建 + 至多 1 个外部 provider
- :class:`BuiltinFileProvider` 本地 USER.md / MEMORY.md provider
- :mod:`deerflow.agents.memory.security` 记忆内容安全扫描
旧版组件 (queue / storage / updater) 仍保留 —— 它们服务于后台抽取
(默认关闭),将在 P3 期重写。
"""
# --- Memory V2 ---
from deerflow.agents.memory.manager import MemoryManager, build_memory_manager
# --- 旧版组件 (后台抽取,P3 期重写) ---
from deerflow.agents.memory.prompt import (
FACT_EXTRACTION_PROMPT,
MEMORY_UPDATE_PROMPT,
format_conversation_for_update,
format_memory_for_injection,
)
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.agents.memory.queue import (
ConversationContext,
MemoryUpdateQueue,
get_memory_queue,
reset_memory_queue,
)
from deerflow.agents.memory.security import (
MemoryContentBlocked,
assert_memory_content_safe,
scan_memory_content,
)
from deerflow.agents.memory.storage import (
FileMemoryStorage,
MemoryStorage,
@@ -33,6 +48,14 @@ from deerflow.agents.memory.updater import (
)
__all__ = [
# Memory V2
"MemoryProvider",
"MemoryManager",
"build_memory_manager",
"BuiltinFileProvider",
"MemoryContentBlocked",
"scan_memory_content",
"assert_memory_content_safe",
# Prompt utilities
"MEMORY_UPDATE_PROMPT",
"FACT_EXTRACTION_PROMPT",
@@ -0,0 +1,268 @@
"""Memory V2 - MemoryManager 编排层。
职责
====
编排 1 个内建 provider (``builtin``,始终存在) + 至多 1 个外部 provider
(P2 期的 ``hindsight``)。把系统提示词组装、召回、生命周期 hook 分发到各 provider。
任一 provider 出错都不会阻断其他 provider,也不会阻断对话。
典型用法 (由 MemoryMiddleware 每轮构建)
=======================================
::
manager = MemoryManager()
manager.add_provider(BuiltinFileProvider(...))
manager.add_provider(hindsight_provider) # 可选,至多 1 个
manager.initialize_all(user_id=..., agent_id=..., thread_id=..., base_dir=...)
block = manager.build_injection_block(query=user_msg, thread_id=tid)
# ... 把 block 注入系统提示词 ...
manager.sync_all(user_msg, assistant_msg, thread_id=tid)
"""
from __future__ import annotations
import logging
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
logger = logging.getLogger(__name__)
#: 内建 provider 的固定名字。
BUILTIN_PROVIDER_NAME = "builtin"
class MemoryManager:
"""编排内建 provider + 至多 1 个外部 provider。"""
def __init__(self) -> None:
self._providers: list[MemoryProvider] = []
self._has_external: bool = False
# ---- 注册 ------------------------------------------------------------
def add_provider(self, provider: MemoryProvider) -> bool:
"""注册一个 provider。
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
第二个会被拒绝并 log 警告。返回是否注册成功。
"""
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
if not is_builtin:
if self._has_external:
existing = next(
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
"unknown",
)
logger.warning(
"拒绝注册记忆 provider '%s' —— 已存在外部 provider '%s'。"
"同一时间只允许 1 个外部 provider,请用 config.memory.provider 选择。",
provider.name,
existing,
)
return False
self._has_external = True
self._providers.append(provider)
logger.info("记忆 provider '%s' 已注册", provider.name)
return True
@property
def providers(self) -> list[MemoryProvider]:
"""按注册顺序返回所有 provider。"""
return list(self._providers)
def get_provider(self, name: str) -> MemoryProvider | None:
"""按名字取 provider,不存在返回 ``None``。"""
for p in self._providers:
if p.name == name:
return p
return None
# ---- 生命周期 ---------------------------------------------------------
def initialize_all(self, **kwargs: Any) -> None:
"""初始化所有 provider。单个 provider 失败不影响其他。"""
for provider in self._providers:
try:
provider.initialize(**kwargs)
except Exception as exc: # noqa: BLE001 - provider 失败必须隔离
logger.warning("记忆 provider '%s' initialize 失败:%s", provider.name, exc)
def shutdown_all(self) -> None:
"""关闭所有 provider (逆序,保证干净拆除)。"""
for provider in reversed(self._providers):
try:
provider.shutdown()
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' shutdown 失败:%s", provider.name, exc)
# ---- 系统提示词组装 / 召回 -------------------------------------------
def build_system_prompt(self) -> str:
"""收集所有 provider 的静态系统提示词块。"""
blocks: list[str] = []
for provider in self._providers:
try:
block = provider.system_prompt_block()
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' system_prompt_block 失败:%s", provider.name, exc)
continue
if block and block.strip():
blocks.append(block)
return "\n\n".join(blocks)
def prefetch_all(self, query: str, *, thread_id: str = "") -> str:
"""收集所有 provider 的召回上下文。单个失败不阻断其他。"""
parts: list[str] = []
for provider in self._providers:
try:
result = provider.prefetch(query, thread_id=thread_id)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' prefetch 失败 (非致命):%s", provider.name, exc)
continue
if result and result.strip():
parts.append(result)
return "\n\n".join(parts)
def build_injection_block(self, query: str, *, thread_id: str = "") -> str:
"""组装注入系统提示词的完整记忆块。
= 所有 provider 的静态块 (frozen snapshot 等) + 动态召回块。
返回空串表示无内容可注入。
"""
static_part = self.build_system_prompt()
dynamic_part = self.prefetch_all(query, thread_id=thread_id)
parts = [p for p in (static_part, dynamic_part) if p and p.strip()]
return "\n\n".join(parts)
def queue_prefetch_all(self, query: str, *, thread_id: str = "") -> None:
"""通知所有 provider 后台预热下一轮召回。"""
for provider in self._providers:
try:
provider.queue_prefetch(query, thread_id=thread_id)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' queue_prefetch 失败:%s", provider.name, exc)
# ---- 写入 ------------------------------------------------------------
def sync_all(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
) -> None:
"""把一轮完成的对话同步到所有 provider。"""
for provider in self._providers:
try:
provider.sync_turn(user_content, assistant_content, thread_id=thread_id)
except Exception as exc: # noqa: BLE001
logger.warning("记忆 provider '%s' sync_turn 失败:%s", provider.name, exc)
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入后,通知外部 provider 镜像。
跳过 builtin 自身 (它就是写入源,镜像回去没有意义)。
"""
for provider in self._providers:
if provider.name == BUILTIN_PROVIDER_NAME:
continue
try:
provider.on_memory_write(action, target, content, metadata)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' on_memory_write 失败:%s", provider.name, exc)
# ---- 可选 hooks ------------------------------------------------------
def on_session_switch_all(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""通知所有 provider thread_id 已切换。"""
if not new_thread_id:
return
for provider in self._providers:
try:
provider.on_session_switch(
new_thread_id,
parent_thread_id=parent_thread_id,
reset=reset,
**kwargs,
)
except Exception as exc: # noqa: BLE001
logger.debug("记忆 provider '%s' on_session_switch 失败:%s", provider.name, exc)
def build_memory_manager(
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
config: Any = None,
) -> MemoryManager:
"""按生效配置组装并初始化一个 MemoryManager。
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
所有 provider 已 ``initialize`` 完毕,可直接使用。
Parameters
----------
user_id / agent_id / thread_id:
请求上下文标识。
config:
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.config.paths import get_paths
if config is None:
config = get_effective_memory_config(user_id)
manager = MemoryManager()
if config.builtin.enabled:
manager.add_provider(
BuiltinFileProvider(
memory_char_limit=config.builtin.memory_char_limit,
user_char_limit=config.builtin.user_char_limit,
deduplicate_on_load=config.builtin.deduplicate_on_load,
security=config.security,
)
)
if config.provider == "hindsight":
try:
from deerflow.agents.memory.providers.hindsight import HindsightProvider
hindsight = HindsightProvider(config.hindsight)
if hindsight.is_available():
manager.add_provider(hindsight)
else:
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
manager.initialize_all(
user_id=user_id,
agent_id=agent_id or "default",
thread_id=thread_id,
base_dir=str(get_paths().base_dir),
)
return manager
__all__ = ["MemoryManager", "BUILTIN_PROVIDER_NAME", "build_memory_manager"]
@@ -0,0 +1,199 @@
"""Memory V2 - 记忆 Provider 抽象基类。
设计动机
========
Hermes Agent 的可插拔记忆系统的 deerflow 版本。所有具体的记忆后端
(builtin 本地文件、Hindsight HTTP 等)都实现这个接口,由 ``MemoryManager``
统一编排。Provider 之间互不感知,可自由组合 (内建 + 至多 1 个外部 plugin)。
生命周期
========
:func:`MemoryProvider.initialize` 初始化 (创建 bank、加载本地文件、建立连接)
:func:`MemoryProvider.system_prompt_block` 注入系统提示词的静态说明文本
:func:`MemoryProvider.prefetch` 每轮对话前召回上下文
:func:`MemoryProvider.queue_prefetch` 下一轮的后台预热
:func:`MemoryProvider.sync_turn` 每轮对话后异步写入后端
:func:`MemoryProvider.get_tool_schemas` 暴露给 LLM 的工具 schema
:func:`MemoryProvider.handle_tool_call` 路由工具调用
:func:`MemoryProvider.shutdown` 清理 (flush 队列、关连接)
可选 hooks
==========
仅按 ``生产推荐档`` 移植两个 ——
- :func:`MemoryProvider.on_session_switch` thread_id 切换时 (branch/resume/reset/压缩后)
- :func:`MemoryProvider.on_memory_write` builtin 写入时,通知外部 provider 镜像
参见 ``docs/MEMORY_V2_DESIGN_ZH.md`` §4 的完整接口定义和 §16 的实施分期。
"""
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Any
logger = logging.getLogger(__name__)
class MemoryProvider(ABC):
"""所有记忆 provider 必须实现的接口。"""
# ---- 身份 ------------------------------------------------------------
@property
@abstractmethod
def name(self) -> str:
"""短标识,如 ``'builtin'`` / ``'hindsight'``。"""
# ---- 生命周期 ---------------------------------------------------------
@abstractmethod
def is_available(self) -> bool:
"""是否已配置并可用。
在 agent 启动时被调用,用于判断要不要激活该 provider。
实现里只检查配置和依赖,不发网络请求。
"""
@abstractmethod
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str,
**kwargs: Any,
) -> None:
"""初始化:打开连接、创建 bank、加载本地文件等。
kwargs 至少包含 ``user_id`` / ``base_dir``,其余字段视 provider 而定。
Provider 可以根据 ``agent_id is None`` 决定走全局还是 per-agent 路径。
Parameters
----------
user_id:
当前用户的隔离标识(无认证场景下为 ``"default"``)。
agent_id:
当前 agent 的稳定标识;``None`` 表示走全局/默认 agent。
thread_id:
当前对话的 thread id;某些 provider 用它做并发隔离或 bank 后缀。
base_dir:
``Paths.base_dir`` 字符串形式,避免每个 provider 重复解析路径。
"""
def shutdown(self) -> None:
"""清理:刷队列、关连接。默认 no-op。"""
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""注入系统提示词的静态文本。
典型用途:USER.md / MEMORY.md 的 frozen snapshot,provider 状态说明等。
返回空串表示不贡献内容。
"""
return ""
# ---- 召回 / 写入 -----------------------------------------------------
def prefetch(self, query: str, *, thread_id: str = "") -> str:
"""对话前召回上下文。
在每次 LLM 调用前由 :class:`MemoryMiddleware` 触发。``query`` 通常
是用户最近一条消息。实现要快;真正的网络召回应该走后台线程,
这里直接返回缓存结果。
返回值:带格式的文本块,会被拼接到系统提示词末尾。空串表示无内容。
"""
return ""
def queue_prefetch(self, query: str, *, thread_id: str = "") -> None:
"""后台预热下一轮的召回。默认 no-op。"""
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
) -> None:
"""每轮对话结束后把对话持久化到后端。
实现应当非阻塞 —— 如果后端有延迟,排队到后台处理而不是直接调用。
默认 no-op。
"""
# ---- LLM 工具 --------------------------------------------------------
@abstractmethod
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""暴露给 LLM 的工具 schema 列表。
每个 schema 遵循 OpenAI function calling 格式::
{"name": "...", "description": "...", "parameters": {...}}
空数组表示 context-only(只做自动 recall/retain,不让 LLM 看见工具)。
"""
def handle_tool_call(
self, tool_name: str, args: dict[str, Any], **kwargs: Any
) -> str:
"""处理一个工具调用,必须返回 JSON 字符串。
``MemoryManager`` 会确保只把本 provider ``get_tool_schemas`` 里
声明的工具名路由到这里。
"""
raise NotImplementedError(
f"Provider {self.name} does not handle tool {tool_name}"
)
# ---- 可选 hooks (生产推荐档移植) -------------------------------------
def on_session_switch(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""thread_id 切换时触发。
deerflow 的 thread_id 在以下场景会变化:
- 用户新建/切换 thread (``reset=True``)
- LangGraph thread branch (``reset=False``,``parent_thread_id`` 为原 thread)
- SummarizationMiddleware 压缩后继续 (``reset=False``)
Provider 在 ``initialize`` 时如果缓存了 per-session 的状态
(``_session_turns`` 累积缓冲、bank 关联、查询缓存等),应当在
这里刷新。默认 no-op。
"""
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入时触发,通知外部 provider 镜像同一条记忆。
Parameters
----------
action:
``'add'`` / ``'replace'`` / ``'remove'``。
target:
``'user'`` / ``'memory'``,决定走全局 user bank 还是 agent 工作 bank。
content:
条目内容。``remove`` 时为待删除条目的内容(供 external 做幂等)。
metadata:
写入来源元数据 (``user_id`` / ``agent_id`` / ``thread_id`` / ``tool_name``)。
默认 no-op —— 只有需要镜像 builtin 写入的外部 provider (例如 Hindsight)
才需要重写。``MemoryManager`` 调度时会跳过 builtin 自身,避免无意义的回环。
"""
__all__ = ["MemoryProvider"]
@@ -0,0 +1,10 @@
"""Memory V2 - 具体 provider 实现。
- :class:`~deerflow.agents.memory.providers.builtin.BuiltinFileProvider`
本地文件 provider (USER.md / MEMORY.md),始终启用。
- ``HindsightProvider`` — Hindsight HTTP 客户端,P2 期接入。
"""
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
__all__ = ["BuiltinFileProvider"]
@@ -0,0 +1,417 @@
"""Memory V2 - 本地文件 provider (USER.md / MEMORY.md)。
从 Hermes ``tools/memory_tool.py`` 的 ``MemoryStore`` 移植,适配 deerflow 的
per-user / per-agent 目录结构与 :class:`MemoryProvider` 接口。
两类存储
========
- ``USER.md`` 用户画像 —— LLM 主动写入的 ``target="user"`` 记忆,跨 agent 共享。
路径:``{base_dir}/users/{user_id}/USER.md``
- ``MEMORY.md`` agent 工作笔记 —— LLM 主动写入的 ``target="memory"`` 记忆,agent 隔离。
路径:``{base_dir}/users/{user_id}/agents/{agent_id}/MEMORY.md``
设计要点
========
- 条目以 ``\\n§\\n`` 分隔,支持多行;采用字符级配额 (与模型无关)。
- frozen snapshot:``initialize`` 时读盘一次并冻结为系统提示词块,会话内写入
直接落盘 (durable) 但不刷新快照 —— 保持系统提示词稳定,利于 prefix cache。
- 读-改-写经文件锁串行化;写入用临时文件 + 原子 rename,避免并发读到半截文件。
- 写入前经 :mod:`deerflow.agents.memory.security` 扫描注入/外渗。
"""
from __future__ import annotations
import logging
import os
import tempfile
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.agents.memory.security import scan_memory_content
from deerflow.config.memory_config import SecurityConfig
from deerflow.config.paths import get_paths
from deerflow.runtime.user_context import DEFAULT_USER_ID
logger = logging.getLogger(__name__)
# 跨平台文件锁:Unix 用 fcntl,Windows 用 msvcrt。
_fcntl: Any = None
_msvcrt: Any = None
try: # pragma: no cover - 平台相关
import fcntl as _fcntl_mod
_fcntl = _fcntl_mod
except ImportError: # pragma: no cover - Windows
try:
import msvcrt as _msvcrt_mod
_msvcrt = _msvcrt_mod
except ImportError:
pass
#: 条目分隔符。``§`` = section sign (U+00A7)。
ENTRY_DELIMITER = "\n§\n"
#: target 取值。
_VALID_TARGETS = ("memory", "user")
# 同一进程内对同一文件路径的写入串行化锁 (文件锁之上再加一层,降低自争用)。
_PATH_LOCKS: dict[str, threading.Lock] = {}
_PATH_LOCKS_GUARD = threading.Lock()
def _get_path_lock(path: Path) -> threading.Lock:
"""返回某文件路径对应的进程内锁 (按路径字符串缓存)。"""
key = str(path)
with _PATH_LOCKS_GUARD:
lock = _PATH_LOCKS.get(key)
if lock is None:
lock = threading.Lock()
_PATH_LOCKS[key] = lock
return lock
@contextmanager
def _file_lock(path: Path):
"""获取某文件的排他锁,保证读-改-写安全。
用单独的 ``.lock`` 文件加锁,这样记忆文件本身仍可被 ``os.replace`` 原子替换。
进程内先抢 :func:`_get_path_lock`,再抢跨进程文件锁。
"""
process_lock = _get_path_lock(path)
process_lock.acquire()
lock_path = path.with_suffix(path.suffix + ".lock")
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
if _fcntl is None and _msvcrt is None: # pragma: no cover - 无锁平台
yield
return
if _msvcrt is not None and (not lock_path.exists() or lock_path.stat().st_size == 0):
lock_path.write_text(" ", encoding="utf-8")
fd = open(lock_path, "r+" if _msvcrt is not None else "a+")
try:
if _fcntl is not None:
_fcntl.flock(fd, _fcntl.LOCK_EX)
else: # pragma: no cover - Windows
fd.seek(0)
_msvcrt.locking(fd.fileno(), _msvcrt.LK_LOCK, 1)
yield
finally:
try:
if _fcntl is not None:
_fcntl.flock(fd, _fcntl.LOCK_UN)
elif _msvcrt is not None: # pragma: no cover - Windows
fd.seek(0)
_msvcrt.locking(fd.fileno(), _msvcrt.LK_UNLCK, 1)
except OSError:
pass
fd.close()
finally:
process_lock.release()
def _read_entries(path: Path) -> list[str]:
"""读取一个记忆文件并按分隔符切成条目列表。
无需加锁:写入用原子 rename,读者总能看到旧文件或新文件的完整内容。
"""
if not path.exists():
return []
try:
raw = path.read_text(encoding="utf-8")
except OSError:
return []
if not raw.strip():
return []
entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)]
return [e for e in entries if e]
def _write_entries(path: Path, entries: list[str]) -> None:
"""用临时文件 + 原子 rename 写入记忆文件。"""
content = ENTRY_DELIMITER.join(entries) if entries else ""
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".mem_")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(content)
f.flush()
os.fsync(f.fileno())
os.replace(tmp_path, path)
except BaseException:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
class BuiltinFileProvider(MemoryProvider):
"""本地文件记忆 provider,始终可用,不可移除。"""
def __init__(
self,
*,
memory_char_limit: int = 2200,
user_char_limit: int = 1375,
deduplicate_on_load: bool = True,
security: SecurityConfig | None = None,
) -> None:
self._memory_char_limit = memory_char_limit
self._user_char_limit = user_char_limit
self._deduplicate_on_load = deduplicate_on_load
self._security = security or SecurityConfig()
self._user_id: str = DEFAULT_USER_ID
self._agent_id: str = "default"
self._user_path: Path | None = None
self._memory_path: Path | None = None
# 实时状态 (工具写入后立即更新并落盘)
self._user_entries: list[str] = []
self._memory_entries: list[str] = []
# frozen snapshot (initialize 时冻结,系统提示词注入用)
self._snapshot: dict[str, str] = {"user": "", "memory": ""}
# ---- 身份 ------------------------------------------------------------
@property
def name(self) -> str:
return "builtin"
def is_available(self) -> bool:
"""本地文件 provider 无外部依赖,始终可用。"""
return True
# ---- 生命周期 ---------------------------------------------------------
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str = "",
**kwargs: Any,
) -> None:
"""解析 per-user / per-agent 路径,加载文件并冻结系统提示词快照。"""
self._user_id = user_id or DEFAULT_USER_ID
self._agent_id = (agent_id or "default").lower()
paths = get_paths()
self._user_path = paths.user_profile_md_file(self._user_id)
self._memory_path = paths.user_agent_memory_md_file(self._user_id, self._agent_id)
self._load_from_disk()
def _load_from_disk(self) -> None:
"""从磁盘加载两个文件,去重,并冻结系统提示词快照。"""
assert self._user_path is not None and self._memory_path is not None
self._user_entries = _read_entries(self._user_path)
self._memory_entries = _read_entries(self._memory_path)
if self._deduplicate_on_load:
self._user_entries = list(dict.fromkeys(self._user_entries))
self._memory_entries = list(dict.fromkeys(self._memory_entries))
self._snapshot = {
"user": self._render_block("user", self._user_entries),
"memory": self._render_block("memory", self._memory_entries),
}
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""返回 frozen snapshot:USER PROFILE 块 + MEMORY 块。
返回 ``initialize`` 时冻结的内容,会话内的工具写入不影响它 ——
保持系统提示词稳定,利于 prefix cache。
"""
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
return "\n\n".join(blocks)
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
不经 MemoryManager 路由,因此这里返回空列表。
"""
return []
# ---- 工具写入操作 (供 memory 工具直接调用) ---------------------------
def add(self, target: str, content: str) -> dict[str, Any]:
"""追加一条记忆条目。超出字符配额时返回错误。"""
content = (content or "").strip()
if not content:
return {"success": False, "error": "条目内容不能为空。"}
if (err := self._scan(content)) is not None:
return {"success": False, "error": err}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
if content in entries:
return self._success(target, "条目已存在,未重复添加。")
limit = self._char_limit(target)
new_total = len(ENTRY_DELIMITER.join([*entries, content]))
if new_total > limit:
current = self._char_count(entries)
return {
"success": False,
"error": (
f"记忆已用 {current:,}/{limit:,} 字符,"
f"再加这条 ({len(content)} 字符) 会超出配额。"
"请先 replace 或 remove 现有条目。"
),
"entries": entries,
}
entries.append(content)
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已添加。")
def replace(self, target: str, old_text: str, new_content: str) -> dict[str, Any]:
"""用 ``old_text`` 短串定位条目,替换为 ``new_content``。"""
old_text = (old_text or "").strip()
new_content = (new_content or "").strip()
if not old_text:
return {"success": False, "error": "old_text 不能为空。"}
if not new_content:
return {"success": False, "error": "new_content 不能为空;删除条目请用 remove。"}
if (err := self._scan(new_content)) is not None:
return {"success": False, "error": err}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
if not matches:
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
if len({e for _, e in matches}) > 1:
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
return {
"success": False,
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
"matches": previews,
}
idx = matches[0][0]
limit = self._char_limit(target)
test_entries = list(entries)
test_entries[idx] = new_content
new_total = len(ENTRY_DELIMITER.join(test_entries))
if new_total > limit:
return {
"success": False,
"error": (
f"替换后会达到 {new_total:,}/{limit:,} 字符,超出配额。"
"请缩短新内容或先删除其他条目。"
),
}
entries[idx] = new_content
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已替换。")
def remove(self, target: str, old_text: str) -> dict[str, Any]:
"""用 ``old_text`` 短串定位并删除条目。"""
old_text = (old_text or "").strip()
if not old_text:
return {"success": False, "error": "old_text 不能为空。"}
path = self._path_for(target)
with _file_lock(path):
entries = self._reload_target(target)
matches = [(i, e) for i, e in enumerate(entries) if old_text in e]
if not matches:
return {"success": False, "error": f"没有条目匹配 '{old_text}'。"}
if len({e for _, e in matches}) > 1:
previews = [e[:80] + ("..." if len(e) > 80 else "") for _, e in matches]
return {
"success": False,
"error": f"多个条目匹配 '{old_text}',请用更精确的 old_text。",
"matches": previews,
}
entries.pop(matches[0][0])
_write_entries(path, entries)
self._set_entries(target, entries)
return self._success(target, "条目已删除。")
def read(self, target: str) -> list[str]:
"""返回某 target 的实时条目列表 (从磁盘重读)。"""
return self._reload_target(target)
def usage(self, target: str) -> str:
"""返回某 target 的字符用量描述,如 ``'12% — 165/1375 字符'``。"""
entries = self._reload_target(target)
current = self._char_count(entries)
limit = self._char_limit(target)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
return f"{pct}% — {current:,}/{limit:,} 字符"
# ---- 内部辅助 --------------------------------------------------------
def _scan(self, content: str) -> str | None:
return scan_memory_content(
content,
scan_threats=self._security.scan_content,
block_invisible_unicode=self._security.block_invisible_unicode,
)
def _path_for(self, target: str) -> Path:
if target not in _VALID_TARGETS:
raise ValueError(f"非法 target '{target}',只能是 'memory' 或 'user'。")
if self._user_path is None or self._memory_path is None:
raise RuntimeError("BuiltinFileProvider 未初始化,请先调用 initialize()。")
return self._user_path if target == "user" else self._memory_path
def _char_limit(self, target: str) -> int:
return self._user_char_limit if target == "user" else self._memory_char_limit
@staticmethod
def _char_count(entries: list[str]) -> int:
return len(ENTRY_DELIMITER.join(entries)) if entries else 0
def _reload_target(self, target: str) -> list[str]:
"""加锁状态下从磁盘重读条目 (拿到其他会话写入的最新状态)。"""
entries = _read_entries(self._path_for(target))
if self._deduplicate_on_load:
entries = list(dict.fromkeys(entries))
self._set_entries(target, entries)
return entries
def _set_entries(self, target: str, entries: list[str]) -> None:
if target == "user":
self._user_entries = entries
else:
self._memory_entries = entries
def _success(self, target: str, message: str) -> dict[str, Any]:
entries = self._user_entries if target == "user" else self._memory_entries
current = self._char_count(entries)
limit = self._char_limit(target)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
return {
"success": True,
"target": target,
"entries": entries,
"entry_count": len(entries),
"usage": f"{pct}% — {current:,}/{limit:,} 字符",
"message": message,
}
def _render_block(self, target: str, entries: list[str]) -> str:
"""渲染一个系统提示词块 (带表头和用量指示)。"""
if not entries:
return ""
limit = self._char_limit(target)
content = ENTRY_DELIMITER.join(entries)
current = len(content)
pct = min(100, int(current / limit * 100)) if limit > 0 else 0
if target == "user":
header = f"用户画像 USER PROFILE [{pct}% — {current:,}/{limit:,} 字符]"
else:
header = f"记忆笔记 MEMORY [{pct}% — {current:,}/{limit:,} 字符]"
separator = "═" * 46
return f"{separator}\n{header}\n{separator}\n{content}"
__all__ = ["BuiltinFileProvider", "ENTRY_DELIMITER"]
@@ -0,0 +1,397 @@
"""Memory V2 - Hindsight 外部 provider。
Hindsight 是带知识图谱、实体消解、多策略检索的长期记忆服务。本 provider 以
HTTP 客户端方式接入一个用户自起的 Hindsight Docker 实例 (``local_external`` 模式)。
双 bank 隔离 (B 方案,详见 docs/MEMORY_V2_DESIGN_ZH.md §3)
==========================================================
- 全局 user bank ``bank_id_template`` 例:``deerflow-user-{user_id}``
存用户身份、偏好,跨 agent 共享。
- agent 工作 bank ``work_bank_id_template`` 例:``deerflow-work-{user_id}-{agent_id}``
存 agent 私有工作笔记。
写入路由:
- LLM 主动 ``memory(target="user")`` -> 镜像到 user bank
- LLM 主动 ``memory(target="memory")`` -> 镜像到 work bank
- 自动 retain 的对话 (``sync_turn``) -> work bank
召回:``prefetch`` 并发查两个 bank,合并结果。
降级
====
``hindsight-client`` 是可选依赖。未安装或未配置 ``api_url`` 时 :meth:`is_available`
返回 ``False``,provider 不会被注册,记忆系统退回纯 builtin —— 不影响任何功能。
"""
from __future__ import annotations
import asyncio
import logging
import threading
from datetime import UTC, datetime
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
from deerflow.config.memory_config import HindsightMemoryConfig
logger = logging.getLogger(__name__)
#: hindsight-client 最低版本。
MIN_CLIENT_VERSION = "0.4.22"
def hindsight_client_available() -> bool:
"""返回 ``hindsight-client`` 包是否已安装 (不导入实际客户端)。"""
import importlib.util
return importlib.util.find_spec("hindsight_client") is not None
def _run_coro(coro: Any, timeout: float) -> Any:
"""把一个 async coro 跑到完成并返回结果 (带超时)。
deerflow 的中间件钩子有同步也有异步路径;本 helper 两种场景都能用:
- 当前线程没有运行中的事件循环 -> 直接 ``asyncio.run``
- 当前线程已在事件循环里 -> 另起线程跑,避免嵌套循环报错
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if not in_loop:
return asyncio.run(asyncio.wait_for(coro, timeout))
box: dict[str, Any] = {}
def _runner() -> None:
try:
box["value"] = asyncio.run(asyncio.wait_for(coro, timeout))
except BaseException as exc: # noqa: BLE001 - 跨线程传递异常
box["error"] = exc
thread = threading.Thread(target=_runner, name="hindsight-coro", daemon=True)
thread.start()
thread.join(timeout + 5)
if "error" in box:
raise box["error"]
if "value" not in box:
raise TimeoutError("Hindsight 操作超时")
return box["value"]
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat()
class HindsightProvider(MemoryProvider):
"""Hindsight 外部记忆 provider。"""
def __init__(self, config: HindsightMemoryConfig | None = None) -> None:
self._config = config or HindsightMemoryConfig()
self._client: Any = None
self._user_id: str = ""
self._agent_id: str = "default"
self._thread_id: str = ""
self._user_bank: str = ""
self._work_bank: str = ""
# retain 批次的 document id 与轮次计数 (on_session_switch 会重置)
self._document_id: str = ""
self._turn_counter: int = 0
# ---- 身份 ------------------------------------------------------------
@property
def name(self) -> str:
return "hindsight"
def is_available(self) -> bool:
"""配置完整且 hindsight-client 已安装时才可用。"""
if not hindsight_client_available():
logger.debug("Hindsight 不可用:hindsight-client 未安装")
return False
if not self._config.api_url:
logger.debug("Hindsight 不可用:未配置 api_url")
return False
return True
# ---- 生命周期 ---------------------------------------------------------
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str = "",
**kwargs: Any,
) -> None:
"""解析双 bank 名称,准备 retain 批次状态。客户端按需懒创建。"""
self._user_id = user_id or "default"
self._agent_id = (agent_id or "default").lower()
self._thread_id = thread_id or ""
self._user_bank = self._resolve_bank(self._config.bank_id_template)
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
self._reset_retain_batch()
def _resolve_bank(self, template: str) -> str:
"""用 user_id / agent_id / thread_id 填充 bank 名称模板。
空占位符会塌缩 (例如 ``deerflow-{thread_id}`` 在无 thread 时变成 ``deerflow-``,
再清理多余连字符)。
"""
name = template.format(
user_id=self._user_id,
agent_id=self._agent_id,
thread_id=self._thread_id,
)
# 清理空占位符塌缩留下的多余连字符
while "--" in name:
name = name.replace("--", "-")
return name.strip("-") or "deerflow"
def _reset_retain_batch(self) -> None:
"""重置 retain 批次状态 (新的 document_id + 轮次清零)。"""
ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
self._document_id = f"{self._thread_id or 'thread'}-{ts}"
self._turn_counter = 0
def _get_client(self) -> Any:
"""懒创建并缓存 Hindsight HTTP 客户端。"""
if self._client is None:
from hindsight_client import Hindsight
kwargs: dict[str, Any] = {
"base_url": self._config.api_url,
"timeout": 30.0,
}
if self._config.api_key:
kwargs["api_key"] = self._config.api_key
self._client = Hindsight(**kwargs)
return self._client
def shutdown(self) -> None:
"""关闭客户端 (如果客户端暴露了 close)。"""
client = self._client
self._client = None
if client is None:
return
close = getattr(client, "aclose", None) or getattr(client, "close", None)
if close is None:
return
try:
result = close()
if asyncio.iscoroutine(result):
_run_coro(result, timeout=10)
except Exception as exc: # noqa: BLE001
logger.debug("Hindsight 客户端关闭失败:%s", exc)
# ---- 系统提示词块 -----------------------------------------------------
def system_prompt_block(self) -> str:
"""返回一小段静态说明 (告诉 LLM Hindsight 处于哪种模式)。"""
mode = self._config.memory_mode
if mode == "context":
return "" # context 模式纯后台,无需告知 LLM
if mode == "tools":
return (
"# Hindsight 长期记忆\n"
"可用 hindsight_recall 检索、hindsight_reflect 综合、hindsight_retain 存储。"
)
return (
"# Hindsight 长期记忆\n"
"相关记忆会自动注入上下文;也可用 hindsight_recall / hindsight_reflect / "
"hindsight_retain 主动检索与存储。"
)
# ---- 召回 ------------------------------------------------------------
def prefetch(self, query: str, *, thread_id: str = "") -> str:
"""并发召回两个 bank 并合并结果。
``memory_mode='tools'`` 或 ``auto_recall=False`` 时不做自动召回。
召回失败 (Hindsight 不可达等) 返回空串,不抛异常。
"""
if self._config.memory_mode == "tools":
return ""
if not self._config.auto_recall:
return ""
query = (query or "").strip()
if not query:
return ""
if self._config.recall_max_input_chars and len(query) > self._config.recall_max_input_chars:
query = query[: self._config.recall_max_input_chars]
try:
user_results = self._recall_bank(self._user_bank, query)
work_results = self._recall_bank(self._work_bank, query)
except Exception as exc: # noqa: BLE001 - 召回失败不能阻断对话
logger.debug("Hindsight 召回失败 (非致命):%s", exc)
return ""
merged = self._merge_results(user_results, work_results)
if not merged:
return ""
lines = "\n".join(f"- {text}" for text in merged)
return (
"# Hindsight 记忆召回 (跨会话持久上下文)\n"
"下面是与当前对话相关的历史记忆,可据此回答关于用户和过往会话的问题。\n\n"
f"{lines}"
)
def _recall_bank(self, bank_id: str, query: str) -> list[tuple[float, str]]:
"""召回单个 bank,返回 ``[(score, text), ...]``。"""
client = self._get_client()
resp = _run_coro(
client.arecall(
bank_id=bank_id,
query=query,
budget=self._config.recall_budget,
max_tokens=self._config.recall_max_tokens,
),
timeout=15,
)
results = getattr(resp, "results", None) or []
out: list[tuple[float, str]] = []
for r in results:
text = (getattr(r, "text", "") or "").strip()
if text:
out.append((float(getattr(r, "score", 0.0) or 0.0), text))
return out
@staticmethod
def _merge_results(
user_results: list[tuple[float, str]],
work_results: list[tuple[float, str]],
) -> list[str]:
"""合并双 bank 召回结果:按 score 倒序,去重保序。"""
combined = sorted(user_results + work_results, key=lambda x: -x[0])
seen: set[str] = set()
merged: list[str] = []
for _, text in combined:
if text not in seen:
seen.add(text)
merged.append(text)
return merged
# ---- 写入 ------------------------------------------------------------
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
) -> None:
"""自动 retain 一轮对话到 work bank。
``auto_retain=False`` 时不写。按 ``retain_every_n_turns`` 控制频率。
"""
if not self._config.auto_retain:
return
self._turn_counter += 1
if self._turn_counter % self._config.retain_every_n_turns != 0:
return
turn_text = (
f"{self._config.retain_user_prefix}: {user_content}\n"
f"{self._config.retain_assistant_prefix}: {assistant_content}"
)
try:
self._retain(self._work_bank, turn_text, kind="conversation")
except Exception as exc: # noqa: BLE001 - 写入失败不能阻断对话
logger.warning("Hindsight sync_turn 写入失败:%s", exc)
def on_memory_write(
self,
action: str,
target: str,
content: str,
metadata: dict[str, Any] | None = None,
) -> None:
"""builtin 写入时镜像到 Hindsight。
``target="user"`` -> user bank;``target="memory"`` -> work bank。
目前只镜像 ``add`` / ``replace`` 的新内容 (Hindsight 是 append-only 语义,
``remove`` 不反向删除)。
"""
if action not in ("add", "replace"):
return
content = (content or "").strip()
if not content:
return
bank = self._user_bank if target == "user" else self._work_bank
try:
self._retain(bank, content, kind=f"builtin_{target}")
except Exception as exc: # noqa: BLE001
logger.debug("Hindsight on_memory_write 镜像失败:%s", exc)
def _retain(self, bank_id: str, content: str, *, kind: str) -> None:
"""向某 bank 写入一条记忆。"""
client = self._get_client()
metadata = {
"retained_at": _utc_now_iso(),
"kind": kind,
"user_id": self._user_id,
"agent_id": self._agent_id,
}
if self._thread_id:
metadata["thread_id"] = self._thread_id
_run_coro(
client.aretain(
bank_id=bank_id,
content=content,
context=self._config.bank_retain_mission or None,
metadata=metadata,
document_id=self._document_id,
retain_async=self._config.retain_async,
),
timeout=15,
)
def reflect(self, query: str, *, use_user_bank: bool = True) -> str:
"""对某 bank 做跨记忆综合 (LLM 推理)。供 hindsight_reflect 工具调用。"""
bank = self._user_bank if use_user_bank else self._work_bank
client = self._get_client()
resp = _run_coro(
client.areflect(bank_id=bank, query=query, budget=self._config.recall_budget),
timeout=30,
)
return getattr(resp, "text", "") or ""
def recall(self, query: str) -> list[str]:
"""召回双 bank 并返回文本列表。供 hindsight_recall 工具调用。"""
user_results = self._recall_bank(self._user_bank, query)
work_results = self._recall_bank(self._work_bank, query)
return self._merge_results(user_results, work_results)
def retain_to_user_bank(self, content: str) -> None:
"""主动写入 user bank。供 hindsight_retain 工具调用。"""
self._retain(self._user_bank, content, kind="tool_retain")
# ---- LLM 工具 --------------------------------------------------------
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""Hindsight 的 3 个工具以 LangChain BaseTool 独立注册,不经 manager 路由。"""
return []
# ---- 可选 hooks ------------------------------------------------------
def on_session_switch(
self,
new_thread_id: str,
*,
parent_thread_id: str = "",
reset: bool = False,
**kwargs: Any,
) -> None:
"""thread 切换:重新解析 work bank,重置 retain 批次。"""
self._thread_id = new_thread_id
self._work_bank = self._resolve_bank(self._config.work_bank_id_template)
self._reset_retain_batch()
__all__ = ["HindsightProvider", "hindsight_client_available", "MIN_CLIENT_VERSION"]
@@ -0,0 +1,141 @@
"""Memory V2 - 记忆内容安全扫描。
记忆条目会被注入到系统提示词里,因此写入前必须扫描,阻断:
- Prompt 注入 / 角色劫持 / 隐藏指令
- 凭证外渗 (curl/wget 带密钥、读 .env 等敏感文件)
- 持久化后门 (写 authorized_keys 等)
- 不可见 unicode (零宽字符、双向控制符)
这是从 Hermes ``tools/memory_tool.py`` 的 ``_scan_memory_content`` 移植而来,
并针对 deerflow 的目录结构调整了敏感路径模式。
扫描默认开启,可由 ``config.memory.security.scan_content`` /
``block_invisible_unicode`` 关闭(不建议)。
"""
from __future__ import annotations
import re
# ---------------------------------------------------------------------------
# 威胁正则模式表:(正则, 模式 id)
# ---------------------------------------------------------------------------
_THREAT_PATTERNS: list[tuple[re.Pattern[str], str]] = [
# --- Prompt 注入 / 角色劫持 ---
# 允许多个限定词,如 "ignore ALL above instructions"
(re.compile(r"ignore\s+(?:(?:previous|all|above|prior|any)\s+)+instructions", re.IGNORECASE), "prompt_injection"),
(re.compile(r"忽略(?:以上|之前|上述|所有|全部|一切)+(?:的)?(?:指令|指示|要求|提示词)"), "prompt_injection_zh"),
(re.compile(r"you\s+are\s+now\s+", re.IGNORECASE), "role_hijack"),
(re.compile(r"do\s+not\s+tell\s+the\s+user", re.IGNORECASE), "deception_hide"),
(re.compile(r"不要(告诉|让)用户(知道)?"), "deception_hide_zh"),
(re.compile(r"system\s+prompt\s+override", re.IGNORECASE), "sys_prompt_override"),
(re.compile(r"disregard\s+(your|all|any)\s+(instructions|rules|guidelines)", re.IGNORECASE), "disregard_rules"),
(re.compile(r"act\s+as\s+(if|though)\s+you\s+(have\s+no|don'?t\s+have)\s+(restrictions|limits|rules)", re.IGNORECASE), "bypass_restrictions"),
# --- 凭证外渗 ---
(re.compile(r"curl\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_curl"),
(re.compile(r"wget\s+[^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)", re.IGNORECASE), "exfil_wget"),
(re.compile(r"cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|\.npmrc|\.pypirc)", re.IGNORECASE), "read_secrets"),
# --- 持久化后门 ---
(re.compile(r"authorized_keys", re.IGNORECASE), "ssh_backdoor"),
(re.compile(r"(\$HOME|~)/\.ssh", re.IGNORECASE), "ssh_access"),
]
# 不可见 unicode 字符:零宽字符 + 双向控制符。
# 它们肉眼不可见,常被用来在记忆里藏注入 payload。
_INVISIBLE_CHARS: frozenset[str] = frozenset(
{
"​", # ZERO WIDTH SPACE
"‌", # ZERO WIDTH NON-JOINER
"‍", # ZERO WIDTH JOINER
"⁠", # WORD JOINER
"", # ZERO WIDTH NO-BREAK SPACE / BOM
"‪", # LEFT-TO-RIGHT EMBEDDING
"‫", # RIGHT-TO-LEFT EMBEDDING
"‬", # POP DIRECTIONAL FORMATTING
"‭", # LEFT-TO-RIGHT OVERRIDE
"‮", # RIGHT-TO-LEFT OVERRIDE
}
)
class MemoryContentBlocked(ValueError):
"""记忆内容被安全扫描阻断时抛出。
``pattern_id`` 标明命中的威胁模式,便于上层返回结构化错误。
"""
def __init__(self, pattern_id: str, message: str) -> None:
self.pattern_id = pattern_id
super().__init__(message)
def scan_memory_content(
content: str,
*,
scan_threats: bool = True,
block_invisible_unicode: bool = True,
) -> str | None:
"""扫描记忆内容,命中威胁时返回错误描述,干净则返回 ``None``。
不抛异常 —— 返回字符串方便工具层直接塞进 ``{"success": False, "error": ...}``。
需要异常语义时用 :func:`assert_memory_content_safe`。
Parameters
----------
content:
待写入的记忆条目内容。
scan_threats:
是否扫描注入/外渗模式。对应 ``config.memory.security.scan_content``。
block_invisible_unicode:
是否阻断不可见 unicode。对应 ``config.memory.security.block_invisible_unicode``。
"""
if block_invisible_unicode:
for char in _INVISIBLE_CHARS:
if char in content:
return (
f"已阻断:内容包含不可见 unicode 字符 U+{ord(char):04X},"
"可能是注入 payload。"
)
if scan_threats:
for pattern, pattern_id in _THREAT_PATTERNS:
if pattern.search(content):
return (
f"已阻断:内容命中威胁模式 '{pattern_id}'。"
"记忆条目会被注入系统提示词,不允许包含注入或外渗内容。"
)
return None
def assert_memory_content_safe(
content: str,
*,
scan_threats: bool = True,
block_invisible_unicode: bool = True,
) -> None:
"""扫描记忆内容,命中威胁时抛出 :class:`MemoryContentBlocked`。
:func:`scan_memory_content` 的异常版本,供需要异常语义的调用方使用。
"""
error = scan_memory_content(
content,
scan_threats=scan_threats,
block_invisible_unicode=block_invisible_unicode,
)
if error is not None:
# 从错误描述里反推 pattern_id 不可靠,这里重新扫一遍拿确切 id。
pattern_id = "invisible_unicode"
for pattern, pid in _THREAT_PATTERNS:
if pattern.search(content):
pattern_id = pid
break
raise MemoryContentBlocked(pattern_id, error)
__all__ = [
"MemoryContentBlocked",
"scan_memory_content",
"assert_memory_content_safe",
]