feat(memory): Memory V2 重写 - 可插拔 provider + Hindsight 集成
以 Hermes 的可插拔记忆架构为骨架重写 deerflow 记忆模块,兼容原有 结构化数据,并新增 Hindsight 作为外部记忆后端。 P1 - 抽象层 + builtin 本地记忆: - MemoryProvider 抽象基类 + MemoryManager 编排层(builtin + ≤1 外部) - BuiltinFileProvider:USER.md/MEMORY.md 二元存储、字符配额、文件锁、 原子写、frozen snapshot - memory 工具(LangChain @tool):LLM 主动写入用户画像/工作笔记 - 记忆内容安全扫描(注入/外渗模式 + 不可见 unicode) - memory_config 改为嵌套结构 + 旧扁平字段兼容 + per-user 三层覆盖 - 迁移脚本 migrate_memory_to_v2.py:旧 memory.json -> USER.md/MEMORY.md - Gateway 新增 /api/memory/v2 与 v2/config 端点(用户可配置白名单字段) P2 - Hindsight 外部 provider: - HindsightProvider:双 bank 隔离(user bank 跨 agent 共享 + work bank agent 私有)、并发召回合并、auto-retain、on_memory_write 镜像 - hindsight_recall/reflect/retain 三个 LangChain 工具 - MemoryMiddleware.before_model 注入 Hindsight 动态召回 - context/tools/hybrid 三种 memory_mode - hindsight-client 作为可选依赖,未装时自动退回纯 builtin 注入走系统提示词(builtin frozen snapshot)+ before_model(Hindsight 动态召回)。后台抽取保留但默认关闭,P3 期重接入。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6947a3ffa7
commit
f9d7610679
21 files changed
+3970
-188
No files matched your search
+389
-54
@@ -1,83 +1,418 @@
|
||||
"""Configuration for memory mechanism."""
|
||||
"""Memory V2 - 记忆子系统配置。
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
配置结构 (详见 docs/MEMORY_V2_DESIGN_ZH.md §9)
|
||||
==============================================
|
||||
``MemoryConfig`` 采用嵌套结构::
|
||||
|
||||
memory:
|
||||
enabled: true
|
||||
provider: hindsight # 外部 provider 选哪个 (builtin 始终启用)
|
||||
builtin: {...} # 本地 USER.md / MEMORY.md
|
||||
hindsight: {...} # Hindsight HTTP 客户端
|
||||
background_extraction: {...} # 后台 LLM 抽取 (默认关)
|
||||
injection: {...} # 系统提示词注入
|
||||
security: {...} # 内容安全扫描
|
||||
|
||||
旧版扁平 yaml (``debounce_seconds`` / ``max_facts`` / ``injection_enabled`` 等)
|
||||
仍可加载:``_migrate_flat_keys`` 校验器会把它们迁移进嵌套结构。
|
||||
|
||||
三层配置覆盖
|
||||
============
|
||||
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
|
||||
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
|
||||
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
|
||||
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
|
||||
4. Pydantic 默认值
|
||||
|
||||
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 嵌套子配置
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class BuiltinMemoryConfig(BaseModel):
|
||||
"""本地文件 provider (USER.md / MEMORY.md) 配置。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="是否启用 builtin 本地记忆 provider")
|
||||
memory_char_limit: int = Field(
|
||||
default=2200, ge=200, le=20000, description="MEMORY.md 字符配额"
|
||||
)
|
||||
user_char_limit: int = Field(
|
||||
default=1375, ge=200, le=20000, description="USER.md 字符配额"
|
||||
)
|
||||
deduplicate_on_load: bool = Field(
|
||||
default=True, description="加载时是否对条目去重"
|
||||
)
|
||||
|
||||
|
||||
class HindsightMemoryConfig(BaseModel):
|
||||
"""Hindsight 外部 provider 配置 (P2 期接入)。"""
|
||||
|
||||
mode: str = Field(
|
||||
default="local_external",
|
||||
description="连接模式:cloud / local_embedded / local_external",
|
||||
)
|
||||
api_url: str = Field(default="http://localhost:8765", description="Hindsight API 地址")
|
||||
api_key: str = Field(default="", description="API key,建议用 $HINDSIGHT_API_KEY 环境变量")
|
||||
bank_id_template: str = Field(
|
||||
default="deerflow-user-{user_id}",
|
||||
description="全局 user bank 命名模板,占位符 {user_id}/{agent_id}/{thread_id}",
|
||||
)
|
||||
work_bank_id_template: str = Field(
|
||||
default="deerflow-work-{user_id}-{agent_id}",
|
||||
description="agent 私有工作 bank 命名模板",
|
||||
)
|
||||
memory_mode: str = Field(
|
||||
default="context",
|
||||
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
|
||||
)
|
||||
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
|
||||
bank_retain_mission: str = Field(default="", description="引导抽取内容的 mission")
|
||||
recall_budget: str = Field(default="mid", description="召回深度:low / mid / high")
|
||||
recall_max_tokens: int = Field(default=4096, ge=256, le=32768, description="召回结果 token 上限")
|
||||
recall_max_input_chars: int = Field(default=800, ge=64, le=8000, description="召回 query 长度上限")
|
||||
auto_recall: bool = Field(default=True, description="每轮对话前是否自动召回")
|
||||
auto_retain: bool = Field(default=True, description="是否自动写入对话")
|
||||
retain_async: bool = Field(default=True, description="retain 是否在 Hindsight 服务端异步处理")
|
||||
retain_every_n_turns: int = Field(default=1, ge=1, le=50, description="每 N 轮 retain 一次")
|
||||
retain_user_prefix: str = Field(default="User", description="自动 retain 时用户发言前缀")
|
||||
retain_assistant_prefix: str = Field(default="Assistant", description="自动 retain 时助手发言前缀")
|
||||
|
||||
|
||||
class BackgroundExtractionConfig(BaseModel):
|
||||
"""后台 LLM 抽取 facts 配置。默认关闭,作为 LLM 漏记的兜底。"""
|
||||
|
||||
enabled: bool = Field(default=False, description="是否启用后台抽取 (默认关)")
|
||||
debounce_seconds: int = Field(
|
||||
default=30, ge=1, le=300, description="抽取前的 debounce 等待秒数"
|
||||
)
|
||||
model_name: str | None = Field(default=None, description="抽取用的模型名 (None = 默认模型)")
|
||||
fact_confidence_threshold: float = Field(
|
||||
default=0.7, ge=0.0, le=1.0, description="fact 入库的最低置信度"
|
||||
)
|
||||
max_facts: int = Field(default=100, ge=10, le=500, description="存储的 fact 数量上限")
|
||||
max_facts_per_extraction: int = Field(
|
||||
default=10, ge=1, le=50, description="单次抽取产出的 fact 上限"
|
||||
)
|
||||
correction_detection: bool = Field(default=True, description="是否检测用户纠错信号")
|
||||
reinforcement_detection: bool = Field(default=True, description="是否检测正反馈信号")
|
||||
|
||||
|
||||
class InjectionConfig(BaseModel):
|
||||
"""系统提示词记忆注入配置。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="是否往系统提示词注入记忆")
|
||||
max_tokens: int = Field(default=2000, ge=100, le=8000, description="注入 token 上限")
|
||||
include_builtin: bool = Field(default=True, description="是否注入 builtin 本地记忆")
|
||||
include_hindsight: bool = Field(default=True, description="是否注入 Hindsight 召回结果")
|
||||
context_tag: str = Field(
|
||||
default="memory-context", description="包裹外部召回内容的标签名"
|
||||
)
|
||||
|
||||
|
||||
class SecurityConfig(BaseModel):
|
||||
"""记忆内容安全配置。"""
|
||||
|
||||
scan_content: bool = Field(default=True, description="是否扫描注入/外渗模式")
|
||||
block_invisible_unicode: bool = Field(default=True, description="是否阻断不可见 unicode")
|
||||
streaming_scrubber: bool = Field(
|
||||
default=True, description="SSE 流出前是否清洗 <memory-context> 标签 (P3)"
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 顶层配置
|
||||
# ===========================================================================
|
||||
|
||||
# 旧版扁平 key -> (嵌套子段, 嵌套字段名) 的迁移映射。
|
||||
_FLAT_KEY_MIGRATION: dict[str, tuple[str, str]] = {
|
||||
"debounce_seconds": ("background_extraction", "debounce_seconds"),
|
||||
"model_name": ("background_extraction", "model_name"),
|
||||
"max_facts": ("background_extraction", "max_facts"),
|
||||
"fact_confidence_threshold": ("background_extraction", "fact_confidence_threshold"),
|
||||
"injection_enabled": ("injection", "enabled"),
|
||||
"max_injection_tokens": ("injection", "max_tokens"),
|
||||
}
|
||||
|
||||
|
||||
class MemoryConfig(BaseModel):
|
||||
"""Configuration for global memory mechanism."""
|
||||
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description="Whether to enable memory mechanism",
|
||||
enabled: bool = Field(default=True, description="记忆子系统总开关")
|
||||
provider: str = Field(
|
||||
default="builtin",
|
||||
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
|
||||
)
|
||||
|
||||
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
|
||||
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
|
||||
background_extraction: BackgroundExtractionConfig = Field(
|
||||
default_factory=BackgroundExtractionConfig
|
||||
)
|
||||
injection: InjectionConfig = Field(default_factory=InjectionConfig)
|
||||
security: SecurityConfig = Field(default_factory=SecurityConfig)
|
||||
|
||||
# --- 旧版扁平字段 (legacy storage.py 专用,V2 不使用,P3 移除) ---
|
||||
storage_path: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Path to store memory data. "
|
||||
"If empty, defaults to per-user memory at `{base_dir}/users/{user_id}/memory.json`. "
|
||||
"Absolute paths are used as-is and opt out of per-user isolation "
|
||||
"(all users share the same file). "
|
||||
"Relative paths are resolved against `Paths.base_dir` "
|
||||
"(not the backend working directory). "
|
||||
"Note: if you previously set this to `.deer-flow/memory.json`, "
|
||||
"the file will now be resolved as `{base_dir}/.deer-flow/memory.json`; "
|
||||
"migrate existing data or use an absolute path to preserve the old location."
|
||||
),
|
||||
description="[旧版] 一体化 memory.json 路径,仅 FileMemoryStorage 使用",
|
||||
)
|
||||
storage_class: str = Field(
|
||||
default="deerflow.agents.memory.storage.FileMemoryStorage",
|
||||
description="The class path for memory storage provider",
|
||||
)
|
||||
debounce_seconds: int = Field(
|
||||
default=30,
|
||||
ge=1,
|
||||
le=300,
|
||||
description="Seconds to wait before processing queued updates (debounce)",
|
||||
)
|
||||
model_name: str | None = Field(
|
||||
default=None,
|
||||
description="Model name to use for memory updates (None = use default model)",
|
||||
)
|
||||
max_facts: int = Field(
|
||||
default=100,
|
||||
ge=10,
|
||||
le=500,
|
||||
description="Maximum number of facts to store",
|
||||
)
|
||||
fact_confidence_threshold: float = Field(
|
||||
default=0.7,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Minimum confidence threshold for storing facts",
|
||||
)
|
||||
injection_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Whether to inject memory into system prompt",
|
||||
)
|
||||
max_injection_tokens: int = Field(
|
||||
default=2000,
|
||||
ge=100,
|
||||
le=8000,
|
||||
description="Maximum tokens to use for memory injection",
|
||||
description="[旧版] 记忆存储类路径,仅 legacy storage 使用",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _migrate_flat_keys(cls, data: Any) -> Any:
|
||||
"""把旧版扁平 yaml 字段迁移进嵌套结构,让旧 config.yaml 仍可加载。"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
data = dict(data) # 不修改调用方的 dict
|
||||
for flat_key, (sub_section, sub_field) in _FLAT_KEY_MIGRATION.items():
|
||||
if flat_key not in data:
|
||||
continue
|
||||
value = data.pop(flat_key)
|
||||
section = data.get(sub_section)
|
||||
if not isinstance(section, dict):
|
||||
section = dict(section) if section else {}
|
||||
# 嵌套结构里已显式给出的值优先,不被扁平值覆盖
|
||||
section.setdefault(sub_field, value)
|
||||
data[sub_section] = section
|
||||
return data
|
||||
|
||||
# --- 兼容属性:让 legacy updater/queue/router/client/prompt 不用改 ---
|
||||
# P3 期 background extraction 重写后,这些 @property 连同调用方一起移除。
|
||||
|
||||
@property
|
||||
def debounce_seconds(self) -> int:
|
||||
"""[兼容] 等价于 ``background_extraction.debounce_seconds``。"""
|
||||
return self.background_extraction.debounce_seconds
|
||||
|
||||
@property
|
||||
def model_name(self) -> str | None:
|
||||
"""[兼容] 等价于 ``background_extraction.model_name``。"""
|
||||
return self.background_extraction.model_name
|
||||
|
||||
@property
|
||||
def max_facts(self) -> int:
|
||||
"""[兼容] 等价于 ``background_extraction.max_facts``。"""
|
||||
return self.background_extraction.max_facts
|
||||
|
||||
@property
|
||||
def fact_confidence_threshold(self) -> float:
|
||||
"""[兼容] 等价于 ``background_extraction.fact_confidence_threshold``。"""
|
||||
return self.background_extraction.fact_confidence_threshold
|
||||
|
||||
@property
|
||||
def injection_enabled(self) -> bool:
|
||||
"""[兼容] 等价于 ``injection.enabled``。"""
|
||||
return self.injection.enabled
|
||||
|
||||
@property
|
||||
def max_injection_tokens(self) -> int:
|
||||
"""[兼容] 等价于 ``injection.max_tokens``。"""
|
||||
return self.injection.max_tokens
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# per-user 运行时覆盖
|
||||
# ===========================================================================
|
||||
|
||||
# 用户可在运行时覆盖的字段白名单 (点号路径)。
|
||||
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
|
||||
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"injection.enabled",
|
||||
"injection.max_tokens",
|
||||
"injection.include_builtin",
|
||||
"injection.include_hindsight",
|
||||
"hindsight.memory_mode",
|
||||
"hindsight.recall_budget",
|
||||
"hindsight.bank_mission",
|
||||
"background_extraction.enabled",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
|
||||
|
||||
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
|
||||
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
def _set_nested(path: str, value: Any) -> None:
|
||||
if path not in USER_WRITABLE_FIELDS:
|
||||
logger.warning("忽略非白名单的记忆配置覆盖字段:%s", path)
|
||||
return
|
||||
section, field = path.split(".", 1)
|
||||
result.setdefault(section, {})[field] = value
|
||||
|
||||
for key, value in overrides.items():
|
||||
if isinstance(value, dict):
|
||||
for sub_key, sub_value in value.items():
|
||||
_set_nested(f"{key}.{sub_key}", sub_value)
|
||||
else:
|
||||
_set_nested(key, value)
|
||||
return result
|
||||
|
||||
|
||||
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
||||
"""递归合并两个 dict,``override`` 优先。返回新 dict,不修改入参。"""
|
||||
merged = copy.deepcopy(base)
|
||||
for key, value in override.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
def _load_json_file(path: Any) -> dict[str, Any]:
|
||||
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return {}
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 全局单例 + 加载入口
|
||||
# ===========================================================================
|
||||
|
||||
# Global configuration instance
|
||||
_memory_config: MemoryConfig = MemoryConfig()
|
||||
|
||||
|
||||
def get_memory_config() -> MemoryConfig:
|
||||
"""Get the current memory configuration."""
|
||||
"""返回 config.yaml 级别的记忆配置 (不含 per-user 覆盖)。"""
|
||||
return _memory_config
|
||||
|
||||
|
||||
def set_memory_config(config: MemoryConfig) -> None:
|
||||
"""Set the memory configuration."""
|
||||
"""设置 config.yaml 级别的记忆配置 (测试与运行时重载用)。"""
|
||||
global _memory_config
|
||||
_memory_config = config
|
||||
|
||||
|
||||
def load_memory_config_from_dict(config_dict: dict) -> None:
|
||||
"""Load memory configuration from a dictionary."""
|
||||
"""从 dict 加载记忆配置 (支持旧扁平 yaml 与新嵌套 yaml)。"""
|
||||
global _memory_config
|
||||
_memory_config = MemoryConfig(**config_dict)
|
||||
|
||||
|
||||
def get_effective_memory_config(user_id: str) -> MemoryConfig:
|
||||
"""按三层覆盖优先级,返回某用户的生效记忆配置。
|
||||
|
||||
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
|
||||
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
|
||||
|
||||
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
|
||||
"""
|
||||
base = _memory_config.model_dump()
|
||||
|
||||
try:
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
paths = get_paths()
|
||||
global_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.memory_global_overrides_file)
|
||||
)
|
||||
user_overrides = _filter_to_whitelist(
|
||||
_load_json_file(paths.user_memory_config_file(user_id))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
|
||||
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
|
||||
try:
|
||||
return MemoryConfig(**merged)
|
||||
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
|
||||
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
|
||||
return _memory_config
|
||||
|
||||
|
||||
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
|
||||
|
||||
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
current = _load_json_file(path)
|
||||
filtered = _filter_to_whitelist(patch)
|
||||
merged = _deep_merge(current, filtered)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return merged
|
||||
|
||||
|
||||
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
|
||||
"""撤销某用户的 per-user 覆盖。
|
||||
|
||||
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
|
||||
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = get_paths().user_memory_config_file(user_id)
|
||||
if field is None:
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
return {}
|
||||
|
||||
current = _load_json_file(path)
|
||||
if "." in field:
|
||||
section, sub_field = field.split(".", 1)
|
||||
if isinstance(current.get(section), dict):
|
||||
current[section].pop(sub_field, None)
|
||||
if not current[section]:
|
||||
current.pop(section, None)
|
||||
else:
|
||||
current.pop(field, None)
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return current
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BuiltinMemoryConfig",
|
||||
"HindsightMemoryConfig",
|
||||
"BackgroundExtractionConfig",
|
||||
"InjectionConfig",
|
||||
"SecurityConfig",
|
||||
"MemoryConfig",
|
||||
"USER_WRITABLE_FIELDS",
|
||||
"get_memory_config",
|
||||
"set_memory_config",
|
||||
"load_memory_config_from_dict",
|
||||
"get_effective_memory_config",
|
||||
"save_user_memory_override",
|
||||
"delete_user_memory_override",
|
||||
]
|
||||
Reference in new issue
Block a user