feat(memory): Memory V2 重写 - 可插拔 provider + Hindsight 集成

以 Hermes 的可插拔记忆架构为骨架重写 deerflow 记忆模块,兼容原有
结构化数据,并新增 Hindsight 作为外部记忆后端。

P1 - 抽象层 + builtin 本地记忆:
- MemoryProvider 抽象基类 + MemoryManager 编排层(builtin + ≤1 外部)
- BuiltinFileProvider:USER.md/MEMORY.md 二元存储、字符配额、文件锁、
  原子写、frozen snapshot
- memory 工具(LangChain @tool):LLM 主动写入用户画像/工作笔记
- 记忆内容安全扫描(注入/外渗模式 + 不可见 unicode)
- memory_config 改为嵌套结构 + 旧扁平字段兼容 + per-user 三层覆盖
- 迁移脚本 migrate_memory_to_v2.py:旧 memory.json -> USER.md/MEMORY.md
- Gateway 新增 /api/memory/v2 与 v2/config 端点(用户可配置白名单字段)

P2 - Hindsight 外部 provider:
- HindsightProvider:双 bank 隔离(user bank 跨 agent 共享 + work bank
  agent 私有)、并发召回合并、auto-retain、on_memory_write 镜像
- hindsight_recall/reflect/retain 三个 LangChain 工具
- MemoryMiddleware.before_model 注入 Hindsight 动态召回
- context/tools/hybrid 三种 memory_mode
- hindsight-client 作为可选依赖,未装时自动退回纯 builtin

注入走系统提示词(builtin frozen snapshot)+ before_model(Hindsight
动态召回)。后台抽取保留但默认关闭,P3 期重接入。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
xuhdandClaude Opus 4.7 committed 2026-05-15 12:15:55 +08:00
1 parent 6947a3ffa7
commit f9d7610679
21 files changed
+3970 -188

No files matched your search

@@ -1,83 +1,418 @@
"""Configuration for memory mechanism."""
"""Memory V2 - 记忆子系统配置。
from pydantic import BaseModel, Field
配置结构 (详见 docs/MEMORY_V2_DESIGN_ZH.md §9)
==============================================
``MemoryConfig`` 采用嵌套结构::
memory:
enabled: true
provider: hindsight # 外部 provider 选哪个 (builtin 始终启用)
builtin: {...} # 本地 USER.md / MEMORY.md
hindsight: {...} # Hindsight HTTP 客户端
background_extraction: {...} # 后台 LLM 抽取 (默认关)
injection: {...} # 系统提示词注入
security: {...} # 内容安全扫描
旧版扁平 yaml (``debounce_seconds`` / ``max_facts`` / ``injection_enabled`` 等)
仍可加载:``_migrate_flat_keys`` 校验器会把它们迁移进嵌套结构。
三层配置覆盖
============
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
4. Pydantic 默认值
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
"""
from __future__ import annotations
import copy
import json
import logging
from typing import Any
from pydantic import BaseModel, Field, model_validator
logger = logging.getLogger(__name__)
# ===========================================================================
# 嵌套子配置
# ===========================================================================
class BuiltinMemoryConfig(BaseModel):
"""本地文件 provider (USER.md / MEMORY.md) 配置。"""
enabled: bool = Field(default=True, description="是否启用 builtin 本地记忆 provider")
memory_char_limit: int = Field(
default=2200, ge=200, le=20000, description="MEMORY.md 字符配额"
)
user_char_limit: int = Field(
default=1375, ge=200, le=20000, description="USER.md 字符配额"
)
deduplicate_on_load: bool = Field(
default=True, description="加载时是否对条目去重"
)
class HindsightMemoryConfig(BaseModel):
"""Hindsight 外部 provider 配置 (P2 期接入)。"""
mode: str = Field(
default="local_external",
description="连接模式:cloud / local_embedded / local_external",
)
api_url: str = Field(default="http://localhost:8765", description="Hindsight API 地址")
api_key: str = Field(default="", description="API key,建议用 $HINDSIGHT_API_KEY 环境变量")
bank_id_template: str = Field(
default="deerflow-user-{user_id}",
description="全局 user bank 命名模板,占位符 {user_id}/{agent_id}/{thread_id}",
)
work_bank_id_template: str = Field(
default="deerflow-work-{user_id}-{agent_id}",
description="agent 私有工作 bank 命名模板",
)
memory_mode: str = Field(
default="context",
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
)
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
bank_retain_mission: str = Field(default="", description="引导抽取内容的 mission")
recall_budget: str = Field(default="mid", description="召回深度:low / mid / high")
recall_max_tokens: int = Field(default=4096, ge=256, le=32768, description="召回结果 token 上限")
recall_max_input_chars: int = Field(default=800, ge=64, le=8000, description="召回 query 长度上限")
auto_recall: bool = Field(default=True, description="每轮对话前是否自动召回")
auto_retain: bool = Field(default=True, description="是否自动写入对话")
retain_async: bool = Field(default=True, description="retain 是否在 Hindsight 服务端异步处理")
retain_every_n_turns: int = Field(default=1, ge=1, le=50, description="每 N 轮 retain 一次")
retain_user_prefix: str = Field(default="User", description="自动 retain 时用户发言前缀")
retain_assistant_prefix: str = Field(default="Assistant", description="自动 retain 时助手发言前缀")
class BackgroundExtractionConfig(BaseModel):
"""后台 LLM 抽取 facts 配置。默认关闭,作为 LLM 漏记的兜底。"""
enabled: bool = Field(default=False, description="是否启用后台抽取 (默认关)")
debounce_seconds: int = Field(
default=30, ge=1, le=300, description="抽取前的 debounce 等待秒数"
)
model_name: str | None = Field(default=None, description="抽取用的模型名 (None = 默认模型)")
fact_confidence_threshold: float = Field(
default=0.7, ge=0.0, le=1.0, description="fact 入库的最低置信度"
)
max_facts: int = Field(default=100, ge=10, le=500, description="存储的 fact 数量上限")
max_facts_per_extraction: int = Field(
default=10, ge=1, le=50, description="单次抽取产出的 fact 上限"
)
correction_detection: bool = Field(default=True, description="是否检测用户纠错信号")
reinforcement_detection: bool = Field(default=True, description="是否检测正反馈信号")
class InjectionConfig(BaseModel):
"""系统提示词记忆注入配置。"""
enabled: bool = Field(default=True, description="是否往系统提示词注入记忆")
max_tokens: int = Field(default=2000, ge=100, le=8000, description="注入 token 上限")
include_builtin: bool = Field(default=True, description="是否注入 builtin 本地记忆")
include_hindsight: bool = Field(default=True, description="是否注入 Hindsight 召回结果")
context_tag: str = Field(
default="memory-context", description="包裹外部召回内容的标签名"
)
class SecurityConfig(BaseModel):
"""记忆内容安全配置。"""
scan_content: bool = Field(default=True, description="是否扫描注入/外渗模式")
block_invisible_unicode: bool = Field(default=True, description="是否阻断不可见 unicode")
streaming_scrubber: bool = Field(
default=True, description="SSE 流出前是否清洗 <memory-context> 标签 (P3)"
)
# ===========================================================================
# 顶层配置
# ===========================================================================
# 旧版扁平 key -> (嵌套子段, 嵌套字段名) 的迁移映射。
_FLAT_KEY_MIGRATION: dict[str, tuple[str, str]] = {
"debounce_seconds": ("background_extraction", "debounce_seconds"),
"model_name": ("background_extraction", "model_name"),
"max_facts": ("background_extraction", "max_facts"),
"fact_confidence_threshold": ("background_extraction", "fact_confidence_threshold"),
"injection_enabled": ("injection", "enabled"),
"max_injection_tokens": ("injection", "max_tokens"),
}
class MemoryConfig(BaseModel):
"""Configuration for global memory mechanism."""
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
enabled: bool = Field(
default=True,
description="Whether to enable memory mechanism",
enabled: bool = Field(default=True, description="记忆子系统总开关")
provider: str = Field(
default="builtin",
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
)
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
background_extraction: BackgroundExtractionConfig = Field(
default_factory=BackgroundExtractionConfig
)
injection: InjectionConfig = Field(default_factory=InjectionConfig)
security: SecurityConfig = Field(default_factory=SecurityConfig)
# --- 旧版扁平字段 (legacy storage.py 专用,V2 不使用,P3 移除) ---
storage_path: str = Field(
default="",
description=(
"Path to store memory data. "
"If empty, defaults to per-user memory at `{base_dir}/users/{user_id}/memory.json`. "
"Absolute paths are used as-is and opt out of per-user isolation "
"(all users share the same file). "
"Relative paths are resolved against `Paths.base_dir` "
"(not the backend working directory). "
"Note: if you previously set this to `.deer-flow/memory.json`, "
"the file will now be resolved as `{base_dir}/.deer-flow/memory.json`; "
"migrate existing data or use an absolute path to preserve the old location."
),
description="[旧版] 一体化 memory.json 路径,仅 FileMemoryStorage 使用",
)
storage_class: str = Field(
default="deerflow.agents.memory.storage.FileMemoryStorage",
description="The class path for memory storage provider",
)
debounce_seconds: int = Field(
default=30,
ge=1,
le=300,
description="Seconds to wait before processing queued updates (debounce)",
)
model_name: str | None = Field(
default=None,
description="Model name to use for memory updates (None = use default model)",
)
max_facts: int = Field(
default=100,
ge=10,
le=500,
description="Maximum number of facts to store",
)
fact_confidence_threshold: float = Field(
default=0.7,
ge=0.0,
le=1.0,
description="Minimum confidence threshold for storing facts",
)
injection_enabled: bool = Field(
default=True,
description="Whether to inject memory into system prompt",
)
max_injection_tokens: int = Field(
default=2000,
ge=100,
le=8000,
description="Maximum tokens to use for memory injection",
description="[旧版] 记忆存储类路径,仅 legacy storage 使用",
)
@model_validator(mode="before")
@classmethod
def _migrate_flat_keys(cls, data: Any) -> Any:
"""把旧版扁平 yaml 字段迁移进嵌套结构,让旧 config.yaml 仍可加载。"""
if not isinstance(data, dict):
return data
data = dict(data) # 不修改调用方的 dict
for flat_key, (sub_section, sub_field) in _FLAT_KEY_MIGRATION.items():
if flat_key not in data:
continue
value = data.pop(flat_key)
section = data.get(sub_section)
if not isinstance(section, dict):
section = dict(section) if section else {}
# 嵌套结构里已显式给出的值优先,不被扁平值覆盖
section.setdefault(sub_field, value)
data[sub_section] = section
return data
# --- 兼容属性:让 legacy updater/queue/router/client/prompt 不用改 ---
# P3 期 background extraction 重写后,这些 @property 连同调用方一起移除。
@property
def debounce_seconds(self) -> int:
"""[兼容] 等价于 ``background_extraction.debounce_seconds``。"""
return self.background_extraction.debounce_seconds
@property
def model_name(self) -> str | None:
"""[兼容] 等价于 ``background_extraction.model_name``。"""
return self.background_extraction.model_name
@property
def max_facts(self) -> int:
"""[兼容] 等价于 ``background_extraction.max_facts``。"""
return self.background_extraction.max_facts
@property
def fact_confidence_threshold(self) -> float:
"""[兼容] 等价于 ``background_extraction.fact_confidence_threshold``。"""
return self.background_extraction.fact_confidence_threshold
@property
def injection_enabled(self) -> bool:
"""[兼容] 等价于 ``injection.enabled``。"""
return self.injection.enabled
@property
def max_injection_tokens(self) -> int:
"""[兼容] 等价于 ``injection.max_tokens``。"""
return self.injection.max_tokens
# ===========================================================================
# per-user 运行时覆盖
# ===========================================================================
# 用户可在运行时覆盖的字段白名单 (点号路径)。
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
{
"injection.enabled",
"injection.max_tokens",
"injection.include_builtin",
"injection.include_hindsight",
"hindsight.memory_mode",
"hindsight.recall_budget",
"hindsight.bank_mission",
"background_extraction.enabled",
}
)
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
"""
result: dict[str, Any] = {}
def _set_nested(path: str, value: Any) -> None:
if path not in USER_WRITABLE_FIELDS:
logger.warning("忽略非白名单的记忆配置覆盖字段:%s", path)
return
section, field = path.split(".", 1)
result.setdefault(section, {})[field] = value
for key, value in overrides.items():
if isinstance(value, dict):
for sub_key, sub_value in value.items():
_set_nested(f"{key}.{sub_key}", sub_value)
else:
_set_nested(key, value)
return result
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
"""递归合并两个 dict,``override`` 优先。返回新 dict,不修改入参。"""
merged = copy.deepcopy(base)
for key, value in override.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _deep_merge(merged[key], value)
else:
merged[key] = value
return merged
def _load_json_file(path: Any) -> dict[str, Any]:
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
try:
from pathlib import Path
p = Path(path)
if not p.exists():
return {}
data = json.loads(p.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (OSError, json.JSONDecodeError) as exc:
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
return {}
# ===========================================================================
# 全局单例 + 加载入口
# ===========================================================================
# Global configuration instance
_memory_config: MemoryConfig = MemoryConfig()
def get_memory_config() -> MemoryConfig:
"""Get the current memory configuration."""
"""返回 config.yaml 级别的记忆配置 (不含 per-user 覆盖)。"""
return _memory_config
def set_memory_config(config: MemoryConfig) -> None:
"""Set the memory configuration."""
"""设置 config.yaml 级别的记忆配置 (测试与运行时重载用)。"""
global _memory_config
_memory_config = config
def load_memory_config_from_dict(config_dict: dict) -> None:
"""Load memory configuration from a dictionary."""
"""从 dict 加载记忆配置 (支持旧扁平 yaml 与新嵌套 yaml)。"""
global _memory_config
_memory_config = MemoryConfig(**config_dict)
def get_effective_memory_config(user_id: str) -> MemoryConfig:
"""按三层覆盖优先级,返回某用户的生效记忆配置。
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
"""
base = _memory_config.model_dump()
try:
from deerflow.config.paths import get_paths
paths = get_paths()
global_overrides = _filter_to_whitelist(
_load_json_file(paths.memory_global_overrides_file)
)
user_overrides = _filter_to_whitelist(
_load_json_file(paths.user_memory_config_file(user_id))
)
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
return _memory_config
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
try:
return MemoryConfig(**merged)
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
return _memory_config
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
current = _load_json_file(path)
filtered = _filter_to_whitelist(patch)
merged = _deep_merge(current, filtered)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
return merged
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
"""撤销某用户的 per-user 覆盖。
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
if field is None:
if path.exists():
path.unlink()
return {}
current = _load_json_file(path)
if "." in field:
section, sub_field = field.split(".", 1)
if isinstance(current.get(section), dict):
current[section].pop(sub_field, None)
if not current[section]:
current.pop(section, None)
else:
current.pop(field, None)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
return current
__all__ = [
"BuiltinMemoryConfig",
"HindsightMemoryConfig",
"BackgroundExtractionConfig",
"InjectionConfig",
"SecurityConfig",
"MemoryConfig",
"USER_WRITABLE_FIELDS",
"get_memory_config",
"set_memory_config",
"load_memory_config_from_dict",
"get_effective_memory_config",
"save_user_memory_override",
"delete_user_memory_override",
]