Files
deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/config/memory_config.py
T
xuhdandClaude Opus 4.7 21a76f5e7c refactor(memory): Memory V2 P3 - 舍弃后台抽取 + Gateway 端点收口
P3 在 P1/P2 基础上完成 Memory V2 收尾:

舍弃后台抽取:
- 删除 updater/queue/storage/prompt/summarization_hook 五个模块
- 移除 background_extraction 配置段、扁平字段兼容、storage 字段
- 理由:Hindsight auto_retain 已自动留存每轮对话,后台抽取冗余

Hindsight 召回注入改用 wrap_model_call:
- 由 before_model 改为 wrap_model_call —— 召回只注入单次模型请求、
  不落持久 state,从根上消除 <memory-context> 泄漏给前端的风险
- builtin frozen snapshot 仍走系统提示词

Gateway 端点收口为 /api/memory/v2*:
- 删除旧 facts/history 端点 (/api/memory、/memory/facts 等)
- 新增 entries 增删改 (POST/PUT/DELETE /memory/v2/entries)
- 新增 POST /memory/v2/search —— Hindsight recall/reflect 检索代理
- client.py memory 方法重写为 V2 等价方法

文档同步更新 CLAUDE.md 与 MEMORY_V2_DESIGN_ZH.md。

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-15 13:08:18 +08:00

316 lines
12 KiB
Python

"""Memory V2 - 记忆子系统配置。
配置结构 (详见 docs/MEMORY_V2_DESIGN_ZH.md §9)
==============================================
``MemoryConfig`` 采用嵌套结构::
memory:
enabled: true
provider: hindsight # 外部 provider 选哪个 (builtin 始终启用)
builtin: {...} # 本地 USER.md / MEMORY.md
hindsight: {...} # Hindsight HTTP 客户端
injection: {...} # 系统提示词注入
security: {...} # 内容安全扫描
三层配置覆盖
============
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
1. per-user 覆盖 ``{base_dir}/users/{user_id}/memory_config.json``
2. 全局覆盖 ``{base_dir}/memory_user_overrides.json``
3. config.yaml ``memory:`` 段 (即 ``get_memory_config()``)
4. Pydantic 默认值
第 1、2 层只允许覆盖 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
"""
from __future__ import annotations
import copy
import json
import logging
from typing import Any
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
# ===========================================================================
# 嵌套子配置
# ===========================================================================
class BuiltinMemoryConfig(BaseModel):
"""本地文件 provider (USER.md / MEMORY.md) 配置。"""
enabled: bool = Field(default=True, description="是否启用 builtin 本地记忆 provider")
memory_char_limit: int = Field(
default=2200, ge=200, le=20000, description="MEMORY.md 字符配额"
)
user_char_limit: int = Field(
default=1375, ge=200, le=20000, description="USER.md 字符配额"
)
deduplicate_on_load: bool = Field(
default=True, description="加载时是否对条目去重"
)
class HindsightMemoryConfig(BaseModel):
"""Hindsight 外部 provider 配置。"""
mode: str = Field(
default="local_external",
description="连接模式:cloud / local_embedded / local_external",
)
api_url: str = Field(default="http://localhost:8765", description="Hindsight API 地址")
api_key: str = Field(default="", description="API key,建议用 $HINDSIGHT_API_KEY 环境变量")
bank_id_template: str = Field(
default="deerflow-user-{user_id}",
description="全局 user bank 命名模板,占位符 {user_id}/{agent_id}/{thread_id}",
)
work_bank_id_template: str = Field(
default="deerflow-work-{user_id}-{agent_id}",
description="agent 私有工作 bank 命名模板",
)
memory_mode: str = Field(
default="context",
description="工具暴露策略:context(仅自动) / tools(仅工具) / hybrid(两者)",
)
bank_mission: str = Field(default="", description="reflect 推理的身份/框架描述")
bank_retain_mission: str = Field(default="", description="引导抽取内容的 mission")
recall_budget: str = Field(default="mid", description="召回深度:low / mid / high")
recall_max_tokens: int = Field(default=4096, ge=256, le=32768, description="召回结果 token 上限")
recall_max_input_chars: int = Field(default=800, ge=64, le=8000, description="召回 query 长度上限")
auto_recall: bool = Field(default=True, description="每轮对话前是否自动召回")
auto_retain: bool = Field(default=True, description="是否自动写入对话")
retain_async: bool = Field(default=True, description="retain 是否在 Hindsight 服务端异步处理")
retain_every_n_turns: int = Field(default=1, ge=1, le=50, description="每 N 轮 retain 一次")
retain_user_prefix: str = Field(default="User", description="自动 retain 时用户发言前缀")
retain_assistant_prefix: str = Field(default="Assistant", description="自动 retain 时助手发言前缀")
class InjectionConfig(BaseModel):
"""系统提示词记忆注入配置。"""
enabled: bool = Field(default=True, description="是否往系统提示词注入记忆")
max_tokens: int = Field(default=2000, ge=100, le=8000, description="注入 token 上限")
include_builtin: bool = Field(default=True, description="是否注入 builtin 本地记忆")
include_hindsight: bool = Field(default=True, description="是否注入 Hindsight 召回结果")
context_tag: str = Field(
default="memory-context", description="包裹外部召回内容的标签名"
)
class SecurityConfig(BaseModel):
"""记忆内容安全配置。"""
scan_content: bool = Field(default=True, description="是否扫描注入/外渗模式")
block_invisible_unicode: bool = Field(default=True, description="是否阻断不可见 unicode")
# ===========================================================================
# 顶层配置
# ===========================================================================
class MemoryConfig(BaseModel):
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
enabled: bool = Field(default=True, description="记忆子系统总开关")
provider: str = Field(
default="builtin",
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
)
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
injection: InjectionConfig = Field(default_factory=InjectionConfig)
security: SecurityConfig = Field(default_factory=SecurityConfig)
# ===========================================================================
# per-user 运行时覆盖
# ===========================================================================
# 用户可在运行时覆盖的字段白名单 (点号路径)。
# 不在此表中的字段必须改 config.yaml 并重启 (例如 provider / api_key / 字符配额)。
USER_WRITABLE_FIELDS: frozenset[str] = frozenset(
{
"injection.enabled",
"injection.max_tokens",
"injection.include_builtin",
"injection.include_hindsight",
"hindsight.memory_mode",
"hindsight.recall_budget",
"hindsight.bank_mission",
}
)
def _filter_to_whitelist(overrides: dict[str, Any]) -> dict[str, Any]:
"""把覆盖 dict 过滤为只含白名单字段的嵌套 dict。
输入既支持嵌套形式 ``{"injection": {"enabled": false}}``,也支持
点号扁平形式 ``{"injection.enabled": false}``。非白名单字段被丢弃并 log。
"""
result: dict[str, Any] = {}
def _set_nested(path: str, value: Any) -> None:
if path not in USER_WRITABLE_FIELDS:
logger.warning("忽略非白名单的记忆配置覆盖字段:%s", path)
return
section, field = path.split(".", 1)
result.setdefault(section, {})[field] = value
for key, value in overrides.items():
if isinstance(value, dict):
for sub_key, sub_value in value.items():
_set_nested(f"{key}.{sub_key}", sub_value)
else:
_set_nested(key, value)
return result
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
"""递归合并两个 dict,``override`` 优先。返回新 dict,不修改入参。"""
merged = copy.deepcopy(base)
for key, value in override.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _deep_merge(merged[key], value)
else:
merged[key] = value
return merged
def _load_json_file(path: Any) -> dict[str, Any]:
"""读取一个 JSON 覆盖文件;不存在或解析失败时返回空 dict。"""
try:
from pathlib import Path
p = Path(path)
if not p.exists():
return {}
data = json.loads(p.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (OSError, json.JSONDecodeError) as exc:
logger.warning("读取记忆配置覆盖文件失败 %s: %s", path, exc)
return {}
# ===========================================================================
# 全局单例 + 加载入口
# ===========================================================================
_memory_config: MemoryConfig = MemoryConfig()
def get_memory_config() -> MemoryConfig:
"""返回 config.yaml 级别的记忆配置 (不含 per-user 覆盖)。"""
return _memory_config
def set_memory_config(config: MemoryConfig) -> None:
"""设置 config.yaml 级别的记忆配置 (测试与运行时重载用)。"""
global _memory_config
_memory_config = config
def load_memory_config_from_dict(config_dict: dict) -> None:
"""从 dict 加载记忆配置。"""
global _memory_config
_memory_config = MemoryConfig(**config_dict)
def get_effective_memory_config(user_id: str) -> MemoryConfig:
"""按三层覆盖优先级,返回某用户的生效记忆配置。
优先级 (高 -> 低):per-user 覆盖 > 全局覆盖 > config.yaml > 默认值。
第 1、2 层只应用 :data:`USER_WRITABLE_FIELDS` 白名单内的字段。
覆盖文件读取失败不会抛异常 —— 退化为只用 config.yaml 级别配置。
"""
base = _memory_config.model_dump()
try:
from deerflow.config.paths import get_paths
paths = get_paths()
global_overrides = _filter_to_whitelist(
_load_json_file(paths.memory_global_overrides_file)
)
user_overrides = _filter_to_whitelist(
_load_json_file(paths.user_memory_config_file(user_id))
)
except Exception as exc: # noqa: BLE001 - 配置覆盖失败必须降级,不能阻断对话
logger.warning("加载 per-user 记忆配置覆盖失败,退回 config.yaml: %s", exc)
return _memory_config
merged = _deep_merge(_deep_merge(base, global_overrides), user_overrides)
try:
return MemoryConfig(**merged)
except Exception as exc: # noqa: BLE001 - 覆盖产生非法配置时降级
logger.warning("合并后的记忆配置非法,退回 config.yaml: %s", exc)
return _memory_config
def save_user_memory_override(user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""把白名单字段写入某用户的 per-user 覆盖文件,返回写入后的完整覆盖 dict。
``patch`` 支持嵌套或点号扁平形式;非白名单字段被忽略。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
current = _load_json_file(path)
filtered = _filter_to_whitelist(patch)
merged = _deep_merge(current, filtered)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
return merged
def delete_user_memory_override(user_id: str, field: str | None = None) -> dict[str, Any]:
"""撤销某用户的 per-user 覆盖。
``field`` 为点号路径 (如 ``injection.enabled``) 时只删该字段;
为 ``None`` 时清空整个 per-user 覆盖文件。返回剩余的覆盖 dict。
"""
from deerflow.config.paths import get_paths
path = get_paths().user_memory_config_file(user_id)
if field is None:
if path.exists():
path.unlink()
return {}
current = _load_json_file(path)
if "." in field:
section, sub_field = field.split(".", 1)
if isinstance(current.get(section), dict):
current[section].pop(sub_field, None)
if not current[section]:
current.pop(section, None)
else:
current.pop(field, None)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
return current
__all__ = [
"BuiltinMemoryConfig",
"HindsightMemoryConfig",
"InjectionConfig",
"SecurityConfig",
"MemoryConfig",
"USER_WRITABLE_FIELDS",
"get_memory_config",
"set_memory_config",
"load_memory_config_from_dict",
"get_effective_memory_config",
"save_user_memory_override",
"delete_user_memory_override",
]