Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xhy
This commit is contained in:
commit
f0ebe81879
157 files changed
+17435
-12710
No files matched your search
+47
-3
@@ -268,6 +268,7 @@ def _build_middlewares(
|
||||
# Add TodoList middleware if plan mode is enabled
|
||||
cfg = _get_runtime_config(config)
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
todo_list_middleware = _create_todo_list_middleware(is_plan_mode)
|
||||
if todo_list_middleware is not None:
|
||||
middlewares.append(todo_list_middleware)
|
||||
@@ -279,8 +280,10 @@ def _build_middlewares(
|
||||
# Add TitleMiddleware
|
||||
middlewares.append(TitleMiddleware(app_config=resolved_app_config))
|
||||
|
||||
# Add MemoryMiddleware (after TitleMiddleware)
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory))
|
||||
# Add MemoryMiddleware (after TitleMiddleware) — skip in bootstrap mode (agent creation flow)
|
||||
memory_injection_enabled = cfg.get("memory_injection_enabled", True)
|
||||
if not is_bootstrap:
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
|
||||
|
||||
# Add ViewImageMiddleware only if the current model supports vision.
|
||||
# Use the resolved runtime model_name from make_lead_agent to avoid stale config values.
|
||||
@@ -309,6 +312,37 @@ def _build_middlewares(
|
||||
|
||||
middlewares.append(SetupAgentApprovalMiddleware())
|
||||
|
||||
# SkillStopMiddleware — intercepts skill invocations and halts execution
|
||||
# when a configured skill is about to run. Request-level options
|
||||
# (skill_stop_names / skill_stop_message) take precedence; static
|
||||
# config in ``app_config.skill_stop`` is the fallback.
|
||||
skill_stop_names = cfg.get("skill_stop_names")
|
||||
skill_stop_message = cfg.get("skill_stop_message")
|
||||
if skill_stop_names:
|
||||
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
|
||||
|
||||
middlewares.append(
|
||||
SkillStopMiddleware(
|
||||
skill_names=skill_stop_names,
|
||||
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
|
||||
stop_message=skill_stop_message,
|
||||
stop_on_match=True,
|
||||
)
|
||||
)
|
||||
logger.info("SkillStopMiddleware enabled (from request): %s", skill_stop_names)
|
||||
elif resolved_app_config.skill_stop.enabled and resolved_app_config.skill_stop.skill_names:
|
||||
from deerflow.agents.middlewares.skill_stop_middleware import SkillStopMiddleware
|
||||
|
||||
middlewares.append(
|
||||
SkillStopMiddleware(
|
||||
skill_names=resolved_app_config.skill_stop.skill_names,
|
||||
skills_root=resolved_app_config.skills.container_path or "/mnt/skills",
|
||||
stop_message=resolved_app_config.skill_stop.stop_message,
|
||||
stop_on_match=resolved_app_config.skill_stop.stop_on_match,
|
||||
)
|
||||
)
|
||||
logger.info("SkillStopMiddleware enabled (from config): %s", resolved_app_config.skill_stop.skill_names)
|
||||
|
||||
# ClarificationMiddleware should always be last
|
||||
middlewares.append(ClarificationMiddleware())
|
||||
return middlewares
|
||||
@@ -335,13 +369,19 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
is_plan_mode = cfg.get("is_plan_mode", False)
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
is_bootstrap = cfg.get("is_bootstrap", False)
|
||||
memory_injection_enabled = bool(cfg.get("memory_injection_enabled", True))
|
||||
writing_mode = bool(cfg.get("writing_mode", False))
|
||||
writing_artifact_path = cfg.get("writing_artifact_path") if writing_mode else None
|
||||
if not isinstance(writing_artifact_path, str) or not writing_artifact_path.strip():
|
||||
writing_artifact_path = None
|
||||
agent_id = validate_agent_id(cfg.get("agent_id") or cfg.get("agent_name"))
|
||||
requested_excluded_tools = cfg.get("excluded_tools") or None
|
||||
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
try:
|
||||
agent_config = load_agent_config(agent_id) if not is_bootstrap else None
|
||||
except FileNotFoundError:
|
||||
logger.warning("Agent directory not found for '%s'; running with default config", agent_id)
|
||||
agent_config = None
|
||||
agent_display_name = agent_config.name if agent_config else agent_id
|
||||
# Custom agent model from agent config (if any), or None to let _resolve_model_name pick the default
|
||||
agent_model_name = agent_config.model if agent_config and agent_config.model else None
|
||||
@@ -443,6 +483,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
model_name=model_name,
|
||||
subagent_enabled=subagent_enabled,
|
||||
is_scheduled_run=cfg.get("is_scheduled_run", False),
|
||||
excluded_tools=requested_excluded_tools,
|
||||
app_config=resolved_app_config,
|
||||
)
|
||||
+ [setup_agent],
|
||||
@@ -455,6 +496,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
available_skills=None,
|
||||
is_bootstrap=True,
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
@@ -470,6 +512,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
groups=effective_tool_groups,
|
||||
subagent_enabled=subagent_enabled,
|
||||
is_scheduled_run=cfg.get("is_scheduled_run", False),
|
||||
excluded_tools=requested_excluded_tools,
|
||||
app_config=resolved_app_config,
|
||||
),
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
@@ -482,6 +525,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
),
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+17
-2
@@ -464,11 +464,22 @@ You are {agent_name}, an internal super agent.
|
||||
- ❌ DO NOT skip clarification for "efficiency" - accuracy matters more than speed
|
||||
- ❌ DO NOT make assumptions when information is missing - ALWAYS ask
|
||||
- ❌ DO NOT proceed with guesses - STOP and call ask_clarification first
|
||||
- ❌ DO NOT cram several unrelated questions into one `question` string — split them into separate tool calls
|
||||
- ✅ Analyze the request in thinking → Identify unclear aspects → Ask BEFORE any action
|
||||
- ✅ If you identify the need for clarification in your thinking, you MUST call the tool IMMEDIATELY
|
||||
- ✅ After calling ask_clarification, execution will be interrupted automatically
|
||||
- ✅ Wait for user response - do NOT continue with assumptions
|
||||
|
||||
**Batching multiple independent questions:**
|
||||
- When the request leaves **several independent aspects** unclear (e.g. project type AND tech stack
|
||||
AND deployment target), call `ask_clarification` multiple times in the same turn — one call per
|
||||
aspect, each with its own focused `question` and `options`.
|
||||
- The frontend groups all clarification cards from one turn into a single submission, so the user
|
||||
answers them together in one reply. You will then receive ONE combined user message addressing
|
||||
every question in order — read it carefully and match answers back to the questions you asked.
|
||||
- If the next question only makes sense after the previous answer (sequential dependency), ask
|
||||
only the first one this turn and wait for the reply before asking the next.
|
||||
|
||||
**How to Use:**
|
||||
```python
|
||||
ask_clarification(
|
||||
@@ -604,7 +615,7 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
|
||||
"""
|
||||
|
||||
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
通过 :class:`MemoryManager` 收集各 provider 的记忆块。P1 期只有 builtin
|
||||
@@ -614,10 +625,13 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
Args:
|
||||
agent_name: agent 标识;``None`` 时按 "default" 处理。
|
||||
app_config: 显式应用配置 (本函数实际按 per-user 覆盖解析生效配置)。
|
||||
injection_enabled: 运行时开关;``False`` 时跳过注入直接返回空串。
|
||||
|
||||
Returns:
|
||||
包在 ``<memory>`` 标签里的记忆上下文;未启用或为空时返回空串。
|
||||
"""
|
||||
if not injection_enabled:
|
||||
return ""
|
||||
try:
|
||||
from deerflow.agents.memory.manager import build_memory_manager
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
@@ -854,11 +868,12 @@ def apply_prompt_template(
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
soul_override: str | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
) -> str:
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
memory_context = _get_memory_context(storage_agent_id, app_config=app_config)
|
||||
memory_context = _get_memory_context(storage_agent_id, app_config=app_config, injection_enabled=memory_injection_enabled)
|
||||
|
||||
# Include subagent section only if enabled (from runtime parameter)
|
||||
n = max_concurrent_subagents
|
||||
|
||||
+42
-21
@@ -223,8 +223,11 @@ def build_memory_manager(
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
这是构建 manager 的统一入口 —— 注册 builtin provider,以及在
|
||||
``config.provider == "hindsight"`` 且 Hindsight 可用时注册 HindsightProvider。
|
||||
这是构建 manager 的统一入口 —— 根据 ``config.version`` 选择 V1 或 V2 provider:
|
||||
|
||||
- ``version="v1"``: 仅注册 V1BuiltinProvider (memory.json + LLM 自动提取)
|
||||
- ``version="v2"``: 注册 BuiltinFileProvider + 可选 HindsightProvider (USER.md/MEMORY.md)
|
||||
|
||||
所有 provider 已 ``initialize`` 完毕,可直接使用。
|
||||
|
||||
Parameters
|
||||
@@ -234,7 +237,6 @@ def build_memory_manager(
|
||||
config:
|
||||
生效的 :class:`MemoryConfig`;为 ``None`` 时按 ``user_id`` 解析 per-user 配置。
|
||||
"""
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
@@ -242,34 +244,53 @@ def build_memory_manager(
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = MemoryManager()
|
||||
base_dir = str(get_paths().base_dir)
|
||||
|
||||
if config.builtin.enabled:
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
v1_cfg = config.v1
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
V1BuiltinProvider(
|
||||
max_facts=v1_cfg.max_facts,
|
||||
fact_confidence_threshold=v1_cfg.fact_confidence_threshold,
|
||||
debounce_seconds=v1_cfg.debounce_seconds,
|
||||
model_name=v1_cfg.model_name,
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
if config.builtin.enabled:
|
||||
manager.add_provider(
|
||||
BuiltinFileProvider(
|
||||
memory_char_limit=config.builtin.memory_char_limit,
|
||||
user_char_limit=config.builtin.user_char_limit,
|
||||
deduplicate_on_load=config.builtin.deduplicate_on_load,
|
||||
security=config.security,
|
||||
)
|
||||
)
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
if config.provider == "hindsight":
|
||||
try:
|
||||
from deerflow.agents.memory.providers.hindsight import HindsightProvider
|
||||
|
||||
hindsight = HindsightProvider(config.hindsight)
|
||||
if hindsight.is_available():
|
||||
manager.add_provider(hindsight)
|
||||
else:
|
||||
logger.info("Hindsight provider 已配置但不可用 (包未装或缺 api_url),退回纯 builtin")
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id or "default",
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
base_dir=str(get_paths().base_dir),
|
||||
base_dir=base_dir,
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
+39
@@ -69,3 +69,42 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
filtered.append(msg)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
_CORRECTION_PATTERNS = re.compile(
|
||||
r"\b(?:no[,.]?\s+(?:that'?s?\s+)?(?:wrong|incorrect|not\s+right|not\s+what)|"
|
||||
r"actually[,.]?\s+(?:it\s+(?:is|should)|you\s+(?:should|need|must)|the\s+correct)|"
|
||||
r"that'?s?\s+(?:wrong|incorrect|not\s+right|not\s+what\s+i)|"
|
||||
r"you\s+(?:got\s+it\s+wrong|misunderstood|made\s+an?\s+error|made\s+a\s+mistake)|"
|
||||
r"incorrect[,.]|wrong[,.]|mistake[,.]|correction:|that\s+is\s+not\s+right)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_REINFORCEMENT_PATTERNS = re.compile(
|
||||
r"\b(?:perfect[,.]?|exactly[,.]?|correct[,.]?|that'?s?\s+(?:right|correct|perfect|exactly\s+right)|"
|
||||
r"you\s+(?:got\s+it|nailed\s+it)|great\s+(?:job|work)|well\s+done|"
|
||||
r"yes[,.]?\s+(?:that'?s?\s+(?:right|correct|exactly)|exactly)|"
|
||||
r"keep\s+(?:it\s+up|doing\s+that|going)|"
|
||||
r"(?:that'?s?\s+|this\s+is\s+)?(?:exactly|precisely)\s+(?:what\s+i|right))\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def detect_correction(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains explicit user correction signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _CORRECTION_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_reinforcement(messages: list[Any]) -> bool:
|
||||
"""Detect if the conversation contains positive reinforcement signals."""
|
||||
for msg in messages:
|
||||
if getattr(msg, "type", None) == "human":
|
||||
text = extract_message_text(msg)
|
||||
if _REINFORCEMENT_PATTERNS.search(text):
|
||||
return True
|
||||
return False
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based MemoryProvider 实现。
|
||||
|
||||
存储格式:``{base_dir}/users/{user_id}/memory.json``(经 FileMemoryStorage 管理)
|
||||
JSON 结构使用嵌套 ``{summary, updatedAt}`` 对象,与原版 DeerFlow 完全一致。
|
||||
|
||||
LLM 在 V1 模式下不调用 memory 工具 —— 事实完全由异步 LLM 提取。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _FakeMsg:
|
||||
"""轻量 message 对象,用于将文本传给原版 format_conversation_for_update。"""
|
||||
__slots__ = ("type", "content")
|
||||
|
||||
def __init__(self, type_: str, content: str) -> None:
|
||||
self.type = type_
|
||||
self.content = content
|
||||
|
||||
|
||||
class V1BuiltinProvider(MemoryProvider):
|
||||
"""原版 DeerFlow V1 记忆 provider。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_facts: int = 100,
|
||||
fact_confidence_threshold: float = 0.7,
|
||||
debounce_seconds: int = 30,
|
||||
model_name: str = "",
|
||||
token_budget: int = 2000,
|
||||
) -> None:
|
||||
self._max_facts = max_facts
|
||||
self._confidence_threshold = fact_confidence_threshold
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._model_name = model_name
|
||||
self._token_budget = token_budget
|
||||
|
||||
self._user_id: str = ""
|
||||
self._thread_id: str = ""
|
||||
self._agent_id: str | None = None
|
||||
self._frozen_block: str = ""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "v1_builtin"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
_ = base_dir # FileMemoryStorage uses get_paths() internally
|
||||
self._user_id = user_id
|
||||
self._thread_id = thread_id or ""
|
||||
self._agent_id = agent_id
|
||||
self._frozen_block = self._build_frozen_block()
|
||||
|
||||
# ---- 系统提示词 -------------------------------------------------------
|
||||
|
||||
def _build_frozen_block(self) -> str:
|
||||
"""从 memory.json 读取数据,用 format_memory_for_injection 渲染注入块。
|
||||
|
||||
当 agent_id 已知时加载 per-agent 记忆;否则加载用户级记忆。
|
||||
"""
|
||||
try:
|
||||
from deerflow.agents.memory.v1.prompt import format_memory_for_injection
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
|
||||
data = get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
text = format_memory_for_injection(data, max_tokens=self._token_budget)
|
||||
if not text:
|
||||
return ""
|
||||
return f"<memory>\n{text}\n</memory>"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider 加载 memory.json 失败: %s", exc)
|
||||
return ""
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
return self._frozen_block
|
||||
|
||||
# ---- 召回 / 写入 ------------------------------------------------------
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
# V1 无动态召回,静态 snapshot 已在 system_prompt_block 里
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
"""把本轮对话加入防抖队列,等待 LLM 异步提取事实。"""
|
||||
if not user_content.strip() and not assistant_content.strip():
|
||||
return
|
||||
try:
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
|
||||
msgs = [
|
||||
_FakeMsg("human", user_content),
|
||||
_FakeMsg("ai", assistant_content),
|
||||
]
|
||||
queue = get_memory_queue()
|
||||
queue.add(
|
||||
thread_id=thread_id or self._thread_id,
|
||||
messages=msgs,
|
||||
user_id=self._user_id,
|
||||
agent_name=self._agent_id or None,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("V1 provider sync_turn 入队失败: %s", exc)
|
||||
|
||||
# ---- 工具 (V1 不暴露工具给 LLM) -------------------------------------
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
# ---- API 读取接口 (供 Gateway router 用) -----------------------------
|
||||
|
||||
def read_v1_data(self) -> dict[str, Any]:
|
||||
"""读取 memory.json 的当前内容(实时读磁盘,非 frozen snapshot)。
|
||||
|
||||
当 agent_id 已知时读取 per-agent 记忆;否则读用户级记忆。
|
||||
"""
|
||||
from deerflow.agents.memory.v1.storage import get_memory_storage
|
||||
return get_memory_storage().load(
|
||||
agent_name=self._agent_id or None,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
|
||||
def fact_count(self) -> int:
|
||||
try:
|
||||
return len(self.read_v1_data().get("facts", []))
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
|
||||
def usage_text(self) -> str:
|
||||
return f"{self.fact_count()}/{self._max_facts} 个事实"
|
||||
|
||||
|
||||
__all__ = ["V1BuiltinProvider"]
|
||||
+1
@@ -0,0 +1 @@
|
||||
"""Memory V1 — 原版 DeerFlow JSON-based 记忆子模块。"""
|
||||
+371
@@ -0,0 +1,371 @@
|
||||
"""Prompt templates for memory update and injection."""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
TIKTOKEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
TIKTOKEN_AVAILABLE = False
|
||||
|
||||
# Prompt template for updating memory based on conversation
|
||||
MEMORY_UPDATE_PROMPT = """You are a memory management system. Your task is to analyze a conversation and update the user's memory profile.
|
||||
|
||||
Current Memory State:
|
||||
<current_memory>
|
||||
{current_memory}
|
||||
</current_memory>
|
||||
|
||||
New Conversation to Process:
|
||||
<conversation>
|
||||
{conversation}
|
||||
</conversation>
|
||||
|
||||
Instructions:
|
||||
1. Analyze the conversation for important information about the user
|
||||
2. Extract relevant facts, preferences, and context with specific details (numbers, names, technologies)
|
||||
3. Update the memory sections as needed following the detailed length guidelines below
|
||||
|
||||
Before extracting facts, perform a structured reflection on the conversation:
|
||||
1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results?
|
||||
If yes, record the root cause and correct approach as a high-confidence fact with category "correction".
|
||||
2. User Correction Detection: Did the user correct the agent's direction, understanding, or output?
|
||||
If yes, record the correct interpretation or approach as a high-confidence fact with category "correction".
|
||||
Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation.
|
||||
3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation?
|
||||
If yes, record them as facts with the most appropriate category and confidence.
|
||||
|
||||
{correction_hint}
|
||||
|
||||
Memory Section Guidelines:
|
||||
|
||||
**User Context** (Current state - concise summaries):
|
||||
- workContext: Professional role, company, key projects, main technologies (2-3 sentences)
|
||||
Example: Core contributor, project names with metrics (16k+ stars), technical stack
|
||||
- personalContext: Languages, communication preferences, key interests (1-2 sentences)
|
||||
Example: Bilingual capabilities, specific interest areas, expertise domains
|
||||
- topOfMind: Multiple ongoing focus areas and priorities (3-5 sentences, detailed paragraph)
|
||||
Example: Primary project work, parallel technical investigations, ongoing learning/tracking
|
||||
Include: Active implementation work, troubleshooting issues, market/research interests
|
||||
Note: This captures SEVERAL concurrent focus areas, not just one task
|
||||
|
||||
**History** (Temporal context - rich paragraphs; updated incrementally each conversation):
|
||||
- recentMonths: Running log of recent work and activities (4-6 sentences or 1-2 paragraphs)
|
||||
Scope: Accumulate from each conversation — any task the user worked on, problems solved, technologies touched
|
||||
Update policy: Set shouldUpdate=true whenever the conversation contains any task, discussion, or work activity; integrate new content into existing text rather than overwriting
|
||||
Include: Technologies explored, projects worked on, problems solved, interests demonstrated
|
||||
- earlierContext: Important patterns promoted from recentMonths over time (3-5 sentences or 1 paragraph)
|
||||
Scope: Recurring themes and established patterns across multiple conversations
|
||||
Update policy: Set shouldUpdate=true when a pattern repeats or an older item in recentMonths should be archived here
|
||||
Include: Past projects, learning journeys, established patterns
|
||||
- longTermBackground: Persistent foundational context that rarely changes (2-4 sentences)
|
||||
Scope: Core expertise, fundamental working style, long-standing interests
|
||||
Update policy: Set shouldUpdate=true only when a genuinely new foundational fact emerges
|
||||
Include: Core expertise, longstanding interests, fundamental working style
|
||||
|
||||
**Facts Extraction**:
|
||||
- Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets")
|
||||
- Include proper nouns (company names, project names, technology names)
|
||||
- Preserve technical terminology and version numbers
|
||||
- Categories:
|
||||
* preference: Tools, styles, approaches user prefers/dislikes
|
||||
* knowledge: Specific expertise, technologies mastered, domain knowledge
|
||||
* context: Background facts (job title, projects, locations, languages)
|
||||
* behavior: Working patterns, communication habits, problem-solving approaches
|
||||
* goal: Stated objectives, learning targets, project ambitions
|
||||
* correction: Explicit agent mistakes or user corrections, including the correct approach
|
||||
- Confidence levels:
|
||||
* 0.9-1.0: Explicitly stated facts ("I work on X", "My role is Y")
|
||||
* 0.7-0.8: Strongly implied from actions/discussions
|
||||
* 0.5-0.6: Inferred patterns (use sparingly, only for clear patterns)
|
||||
|
||||
**What Goes Where**:
|
||||
- workContext: Current job, active projects, primary tech stack
|
||||
- personalContext: Languages, personality, interests outside direct work tasks
|
||||
- topOfMind: Multiple ongoing priorities and focus areas user cares about recently (gets updated most frequently)
|
||||
Should capture 3-5 concurrent themes: main work, side explorations, learning/tracking interests
|
||||
- recentMonths: Cumulative log of recent work — update from EVERY conversation with meaningful content
|
||||
- earlierContext: Recurring patterns promoted from recentMonths; update when patterns solidify
|
||||
- longTermBackground: Unchanging foundational facts about the user; update only for new foundational info
|
||||
|
||||
**Language Requirement**:
|
||||
- ALL summaries and fact content MUST be written in Chinese (Simplified)
|
||||
- Keep technical terms, proper nouns, product names, and company names in their original form (e.g., DeepSeek, LangGraph, GitHub)
|
||||
- Surrounding descriptive text must be in Chinese regardless of the conversation language
|
||||
|
||||
Output Format (JSON):
|
||||
{{
|
||||
"user": {{
|
||||
"workContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"personalContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"topOfMind": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"history": {{
|
||||
"recentMonths": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"earlierContext": {{ "summary": "...", "shouldUpdate": true/false }},
|
||||
"longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }}
|
||||
}},
|
||||
"newFacts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
],
|
||||
"factsToRemove": ["fact_id_1", "fact_id_2"]
|
||||
}}
|
||||
|
||||
Important Rules:
|
||||
- For recentMonths: set shouldUpdate=true whenever the conversation contains any task, work, or substantive discussion — this field accumulates like a log
|
||||
- For earlierContext / longTermBackground: set shouldUpdate=true only when a new pattern solidifies or a foundational fact changes
|
||||
- For user context fields: set shouldUpdate=true if there's meaningful new information
|
||||
- Follow length guidelines: workContext/personalContext are concise (1-3 sentences), topOfMind and history sections are detailed (paragraphs)
|
||||
- Include specific metrics, version numbers, and proper nouns in facts
|
||||
- Only add facts that are clearly stated (0.9+) or strongly implied (0.7+)
|
||||
- Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit
|
||||
- Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise
|
||||
- Remove facts that are contradicted by new information
|
||||
- When updating topOfMind, integrate new focus areas while removing completed/abandoned ones
|
||||
Keep 3-5 concurrent focus themes that are still active and relevant
|
||||
- For history sections, integrate new information chronologically into appropriate time period
|
||||
- Preserve technical accuracy - keep exact names of technologies, companies, projects
|
||||
- Focus on information useful for future interactions and personalization
|
||||
- IMPORTANT: Do NOT record file upload events in memory. Uploaded files are
|
||||
session-specific and ephemeral — they will not be accessible in future sessions.
|
||||
Recording upload events causes confusion in subsequent conversations.
|
||||
- CRITICAL: Write ALL summary text and fact content in Chinese (Simplified). Technical terms and proper nouns (product names, company names, technology names) may remain in their original language, but all other text must be in Chinese.
|
||||
|
||||
Return ONLY valid JSON, no explanation or markdown."""
|
||||
|
||||
|
||||
# Prompt template for extracting facts from a single message
|
||||
FACT_EXTRACTION_PROMPT = """Extract factual information about the user from this message.
|
||||
|
||||
Message:
|
||||
{message}
|
||||
|
||||
Extract facts in this JSON format:
|
||||
{{
|
||||
"facts": [
|
||||
{{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0 }}
|
||||
]
|
||||
}}
|
||||
|
||||
Categories:
|
||||
- preference: User preferences (likes/dislikes, styles, tools)
|
||||
- knowledge: User's expertise or knowledge areas
|
||||
- context: Background context (location, job, projects)
|
||||
- behavior: Behavioral patterns
|
||||
- goal: User's goals or objectives
|
||||
- correction: Explicit corrections or mistakes to avoid repeating
|
||||
|
||||
Rules:
|
||||
- Only extract clear, specific facts
|
||||
- Confidence should reflect certainty (explicit statement = 0.9+, implied = 0.6-0.8)
|
||||
- Skip vague or temporary information
|
||||
- Write all fact content in Chinese (Simplified); technical terms and proper nouns may remain in their original language
|
||||
|
||||
Return ONLY valid JSON."""
|
||||
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
|
||||
"""Count tokens in text using tiktoken.
|
||||
|
||||
Args:
|
||||
text: The text to count tokens for.
|
||||
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
|
||||
|
||||
Returns:
|
||||
The number of tokens in the text.
|
||||
"""
|
||||
if not TIKTOKEN_AVAILABLE:
|
||||
return len(text) // 4
|
||||
|
||||
try:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
return len(encoding.encode(text))
|
||||
except Exception:
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def _coerce_confidence(value: Any, default: float = 0.0) -> float:
|
||||
"""Coerce a confidence-like value to a bounded float in [0, 1].
|
||||
|
||||
Non-finite values (NaN, inf, -inf) are treated as invalid and fall back
|
||||
to the default before clamping, preventing them from dominating ranking.
|
||||
The ``default`` parameter is assumed to be a finite value.
|
||||
"""
|
||||
try:
|
||||
confidence = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return max(0.0, min(1.0, default))
|
||||
if not math.isfinite(confidence):
|
||||
return max(0.0, min(1.0, default))
|
||||
return max(0.0, min(1.0, confidence))
|
||||
|
||||
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
|
||||
"""Format memory data for injection into system prompt.
|
||||
|
||||
Args:
|
||||
memory_data: The memory data dictionary.
|
||||
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
|
||||
|
||||
Returns:
|
||||
Formatted memory string for system prompt injection.
|
||||
"""
|
||||
if not memory_data:
|
||||
return ""
|
||||
|
||||
sections = []
|
||||
|
||||
# Format user context
|
||||
user_data = memory_data.get("user", {})
|
||||
if user_data:
|
||||
user_sections = []
|
||||
|
||||
work_ctx = user_data.get("workContext", {})
|
||||
if work_ctx.get("summary"):
|
||||
user_sections.append(f"Work: {work_ctx['summary']}")
|
||||
|
||||
personal_ctx = user_data.get("personalContext", {})
|
||||
if personal_ctx.get("summary"):
|
||||
user_sections.append(f"Personal: {personal_ctx['summary']}")
|
||||
|
||||
top_of_mind = user_data.get("topOfMind", {})
|
||||
if top_of_mind.get("summary"):
|
||||
user_sections.append(f"Current Focus: {top_of_mind['summary']}")
|
||||
|
||||
if user_sections:
|
||||
sections.append("User Context:\n" + "\n".join(f"- {s}" for s in user_sections))
|
||||
|
||||
# Format history
|
||||
history_data = memory_data.get("history", {})
|
||||
if history_data:
|
||||
history_sections = []
|
||||
|
||||
recent = history_data.get("recentMonths", {})
|
||||
if recent.get("summary"):
|
||||
history_sections.append(f"Recent: {recent['summary']}")
|
||||
|
||||
earlier = history_data.get("earlierContext", {})
|
||||
if earlier.get("summary"):
|
||||
history_sections.append(f"Earlier: {earlier['summary']}")
|
||||
|
||||
background = history_data.get("longTermBackground", {})
|
||||
if background.get("summary"):
|
||||
history_sections.append(f"Background: {background['summary']}")
|
||||
|
||||
if history_sections:
|
||||
sections.append("History:\n" + "\n".join(f"- {s}" for s in history_sections))
|
||||
|
||||
# Format facts (sorted by confidence; include as many as token budget allows)
|
||||
facts_data = memory_data.get("facts", [])
|
||||
if isinstance(facts_data, list) and facts_data:
|
||||
ranked_facts = sorted(
|
||||
(f for f in facts_data if isinstance(f, dict) and isinstance(f.get("content"), str) and f.get("content").strip()),
|
||||
key=lambda fact: _coerce_confidence(fact.get("confidence"), default=0.0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
base_text = "\n\n".join(sections)
|
||||
base_tokens = _count_tokens(base_text) if base_text else 0
|
||||
facts_header = "Facts:\n"
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
|
||||
running_tokens = base_tokens + separator_tokens
|
||||
|
||||
fact_lines: list[str] = []
|
||||
for fact in ranked_facts:
|
||||
content_value = fact.get("content")
|
||||
if not isinstance(content_value, str):
|
||||
continue
|
||||
content = content_value.strip()
|
||||
if not content:
|
||||
continue
|
||||
category = str(fact.get("category", "context")).strip() or "context"
|
||||
confidence = _coerce_confidence(fact.get("confidence"), default=0.0)
|
||||
source_error = fact.get("sourceError")
|
||||
if category == "correction" and isinstance(source_error, str) and source_error.strip():
|
||||
line = f"- [{category} | {confidence:.2f}] {content} (avoid: {source_error.strip()})"
|
||||
else:
|
||||
line = f"- [{category} | {confidence:.2f}] {content}"
|
||||
|
||||
line_text = ("\n" + line) if fact_lines else line
|
||||
line_tokens = _count_tokens(line_text)
|
||||
|
||||
if running_tokens + line_tokens <= max_tokens:
|
||||
fact_lines.append(line)
|
||||
running_tokens += line_tokens
|
||||
else:
|
||||
break
|
||||
|
||||
if fact_lines:
|
||||
sections.append("Facts:\n" + "\n".join(fact_lines))
|
||||
|
||||
if not sections:
|
||||
return ""
|
||||
|
||||
result = "\n\n".join(sections)
|
||||
|
||||
token_count = _count_tokens(result)
|
||||
if token_count > max_tokens:
|
||||
char_per_token = len(result) / token_count
|
||||
target_chars = int(max_tokens * char_per_token * 0.95)
|
||||
result = result[:target_chars] + "\n..."
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_conversation_for_update(messages: list[Any]) -> str:
|
||||
"""Format conversation messages for memory update prompt.
|
||||
|
||||
Args:
|
||||
messages: List of conversation messages.
|
||||
|
||||
Returns:
|
||||
Formatted conversation string.
|
||||
"""
|
||||
lines = []
|
||||
for msg in messages:
|
||||
role = getattr(msg, "type", "unknown")
|
||||
content = getattr(msg, "content", str(msg))
|
||||
|
||||
# Handle content that might be a list (multimodal)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for p in content:
|
||||
if isinstance(p, str):
|
||||
text_parts.append(p)
|
||||
elif isinstance(p, dict):
|
||||
text_val = p.get("text")
|
||||
if isinstance(text_val, str):
|
||||
text_parts.append(text_val)
|
||||
content = " ".join(text_parts) if text_parts else str(content)
|
||||
|
||||
# Strip uploaded_files tags from human messages to avoid persisting
|
||||
# ephemeral file path info into long-term memory. Skip the turn entirely
|
||||
# when nothing remains after stripping (upload-only message).
|
||||
if role == "human":
|
||||
content = re.sub(r"<uploaded_files>[\s\S]*?</uploaded_files>\n*", "", str(content)).strip()
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Truncate very long messages
|
||||
if len(str(content)) > 1000:
|
||||
content = str(content)[:1000] + "..."
|
||||
|
||||
if role == "human":
|
||||
lines.append(f"User: {content}")
|
||||
elif role == "ai":
|
||||
lines.append(f"Assistant: {content}")
|
||||
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MEMORY_UPDATE_PROMPT",
|
||||
"FACT_EXTRACTION_PROMPT",
|
||||
"format_memory_for_injection",
|
||||
"format_conversation_for_update",
|
||||
"_count_tokens",
|
||||
"_coerce_confidence",
|
||||
]
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Memory V1 — 防抖异步更新队列(原版 DeerFlow MemoryUpdateQueue)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
def _memory_enabled() -> bool:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().enabled
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationContext:
|
||||
"""Context for a conversation to be processed for memory update."""
|
||||
thread_id: str
|
||||
messages: list[Any]
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
agent_name: str | None = None
|
||||
user_id: str | None = None
|
||||
correction_detected: bool = False
|
||||
reinforcement_detected: bool = False
|
||||
|
||||
|
||||
class MemoryUpdateQueue:
|
||||
"""Queue for memory updates with debounce mechanism.
|
||||
|
||||
Collects conversation contexts and processes them after a configurable
|
||||
debounce period. Multiple conversations received within the debounce
|
||||
window are deduplicated by (thread_id, user_id, agent_name).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue: list[ConversationContext] = []
|
||||
self._lock = threading.Lock()
|
||||
self._timer: threading.Timer | None = None
|
||||
self._processing = False
|
||||
|
||||
@staticmethod
|
||||
def _queue_key(thread_id: str, user_id: str | None, agent_name: str | None) -> tuple[str, str | None, str | None]:
|
||||
return (thread_id, user_id, agent_name)
|
||||
|
||||
def add(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation to the update queue (with debounce)."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._reset_timer()
|
||||
|
||||
logger.info("V1 memory update queued for thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def add_nowait(
|
||||
self,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
) -> None:
|
||||
"""Add a conversation and start processing immediately in the background."""
|
||||
if not _memory_enabled():
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
self._enqueue_locked(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
self._schedule_timer(0)
|
||||
|
||||
logger.info("V1 memory update queued for immediate processing on thread %s, queue size: %d", thread_id, len(self._queue))
|
||||
|
||||
def _enqueue_locked(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
user_id: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
) -> None:
|
||||
queue_key = self._queue_key(thread_id, user_id, agent_name)
|
||||
|
||||
existing = next(
|
||||
(ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) == queue_key),
|
||||
None,
|
||||
)
|
||||
|
||||
merged_correction = correction_detected or (existing.correction_detected if existing is not None else False)
|
||||
merged_reinforcement = reinforcement_detected or (existing.reinforcement_detected if existing is not None else False)
|
||||
|
||||
context = ConversationContext(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=merged_correction,
|
||||
reinforcement_detected=merged_reinforcement,
|
||||
)
|
||||
|
||||
self._queue = [ctx for ctx in self._queue if self._queue_key(ctx.thread_id, ctx.user_id, ctx.agent_name) != queue_key]
|
||||
self._queue.append(context)
|
||||
|
||||
def _reset_timer(self) -> None:
|
||||
cfg = _get_v1_cfg()
|
||||
self._schedule_timer(cfg.debounce_seconds)
|
||||
logger.debug("V1 memory update timer set for %ss", cfg.debounce_seconds)
|
||||
|
||||
def _schedule_timer(self, delay_seconds: float) -> None:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = threading.Timer(delay_seconds, self._process_queue)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _process_queue(self) -> None:
|
||||
from deerflow.agents.memory.v1.updater import MemoryUpdater
|
||||
|
||||
with self._lock:
|
||||
if self._processing:
|
||||
self._schedule_timer(0)
|
||||
return
|
||||
if not self._queue:
|
||||
return
|
||||
self._processing = True
|
||||
contexts_to_process = self._queue.copy()
|
||||
self._queue.clear()
|
||||
self._timer = None
|
||||
|
||||
logger.info("V1 memory: processing %d queued updates", len(contexts_to_process))
|
||||
|
||||
try:
|
||||
updater = MemoryUpdater()
|
||||
for context in contexts_to_process:
|
||||
try:
|
||||
logger.info("V1 memory: updating for thread %s", context.thread_id)
|
||||
success = updater.update_memory(
|
||||
messages=context.messages,
|
||||
thread_id=context.thread_id,
|
||||
agent_name=context.agent_name,
|
||||
correction_detected=context.correction_detected,
|
||||
reinforcement_detected=context.reinforcement_detected,
|
||||
user_id=context.user_id,
|
||||
)
|
||||
if success:
|
||||
logger.info("V1 memory: updated successfully for thread %s", context.thread_id)
|
||||
else:
|
||||
logger.warning("V1 memory: update skipped/failed for thread %s", context.thread_id)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error("V1 memory: error updating for thread %s: %s", context.thread_id, e)
|
||||
|
||||
if len(contexts_to_process) > 1:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processing = False
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force immediate processing of the queue."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._process_queue()
|
||||
|
||||
def flush_nowait(self) -> None:
|
||||
"""Start queue processing immediately in a background thread."""
|
||||
with self._lock:
|
||||
self._schedule_timer(0)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear the queue without processing (for testing)."""
|
||||
with self._lock:
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
self._queue.clear()
|
||||
self._processing = False
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._queue)
|
||||
|
||||
@property
|
||||
def is_processing(self) -> bool:
|
||||
with self._lock:
|
||||
return self._processing
|
||||
|
||||
|
||||
_memory_queue: MemoryUpdateQueue | None = None
|
||||
_queue_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_queue() -> MemoryUpdateQueue:
|
||||
"""Get the global memory update queue singleton."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is None:
|
||||
_memory_queue = MemoryUpdateQueue()
|
||||
return _memory_queue
|
||||
|
||||
|
||||
def reset_memory_queue() -> None:
|
||||
"""Reset the global memory queue (for testing)."""
|
||||
global _memory_queue
|
||||
with _queue_lock:
|
||||
if _memory_queue is not None:
|
||||
_memory_queue.clear()
|
||||
_memory_queue = None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConversationContext",
|
||||
"MemoryUpdateQueue",
|
||||
"get_memory_queue",
|
||||
"reset_memory_queue",
|
||||
]
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
"""Memory V1 — 存储层(原版 DeerFlow 格式)。
|
||||
|
||||
JSON 结构:
|
||||
{
|
||||
"version": "1.0",
|
||||
"lastUpdated": "...",
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground":{"summary": "", "updatedAt": ""}
|
||||
},
|
||||
"facts": []
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def utc_now_iso_z() -> str:
|
||||
"""Current UTC time as ISO-8601 with ``Z`` suffix."""
|
||||
return datetime.now(UTC).isoformat().removesuffix("+00:00") + "Z"
|
||||
|
||||
|
||||
def create_empty_memory() -> dict[str, Any]:
|
||||
"""Create an empty memory structure (new format)."""
|
||||
return {
|
||||
"version": "1.0",
|
||||
"lastUpdated": utc_now_iso_z(),
|
||||
"user": {
|
||||
"workContext": {"summary": "", "updatedAt": ""},
|
||||
"personalContext": {"summary": "", "updatedAt": ""},
|
||||
"topOfMind": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"history": {
|
||||
"recentMonths": {"summary": "", "updatedAt": ""},
|
||||
"earlierContext": {"summary": "", "updatedAt": ""},
|
||||
"longTermBackground": {"summary": "", "updatedAt": ""},
|
||||
},
|
||||
"facts": [],
|
||||
}
|
||||
|
||||
|
||||
def _migrate_legacy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把老格式 {userContext: {workContext: str}} 迁移到新格式 {user: {workContext: {summary, updatedAt}}}。"""
|
||||
if "userContext" not in data:
|
||||
return data
|
||||
|
||||
now = utc_now_iso_z()
|
||||
old_ctx = data.pop("userContext", {})
|
||||
data["user"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in old_ctx.items()
|
||||
}
|
||||
|
||||
# history 字段可能是纯字符串,需要包裹
|
||||
if "history" in data:
|
||||
hist = data["history"]
|
||||
data["history"] = {
|
||||
k: {"summary": v or "", "updatedAt": now} if isinstance(v, str) else v
|
||||
for k, v in hist.items()
|
||||
}
|
||||
|
||||
# 确保 user 包含所有必要字段
|
||||
base = create_empty_memory()
|
||||
for key in ("user", "history"):
|
||||
section = base[key]
|
||||
for field in section:
|
||||
if field not in data.get(key, {}):
|
||||
data.setdefault(key, {})[field] = section[field]
|
||||
|
||||
data["version"] = "1.0"
|
||||
data.pop("updatedAt", None)
|
||||
return data
|
||||
|
||||
|
||||
class FileMemoryStorage:
|
||||
"""File-based memory storage provider (原版 DeerFlow 实现)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Per-(user_id, agent_name) cache: value = (memory_data, file_mtime)
|
||||
self._memory_cache: dict[tuple[str | None, str | None], tuple[dict[str, Any], float | None]] = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
|
||||
def _validate_agent_name(self, agent_name: str) -> None:
|
||||
if not agent_name:
|
||||
raise ValueError("Agent name must be a non-empty string.")
|
||||
if not AGENT_NAME_PATTERN.match(agent_name):
|
||||
raise ValueError(f"Invalid agent name {agent_name!r}: names must match {AGENT_NAME_PATTERN.pattern}")
|
||||
|
||||
def _get_memory_file_path(self, agent_name: str | None = None, *, user_id: str | None = None) -> Path:
|
||||
paths = get_paths()
|
||||
if user_id is not None:
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.user_agent_memory_file(user_id, agent_name)
|
||||
return paths.user_memory_file(user_id)
|
||||
# Legacy: no user_id
|
||||
if agent_name is not None:
|
||||
self._validate_agent_name(agent_name)
|
||||
return paths.agent_memory_file(agent_name)
|
||||
return paths.memory_file
|
||||
|
||||
def _load_from_file(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
if not file_path.exists():
|
||||
return create_empty_memory()
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return create_empty_memory()
|
||||
data = _migrate_legacy(data)
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning("Failed to load memory file %s: %s", file_path, e)
|
||||
return create_empty_memory()
|
||||
|
||||
@staticmethod
|
||||
def _cache_key(agent_name: str | None = None, *, user_id: str | None = None) -> tuple[str | None, str | None]:
|
||||
return (user_id, agent_name)
|
||||
|
||||
def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Load memory data (cached with file modification time check)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
current_mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
current_mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
cached = self._memory_cache.get(cache_key)
|
||||
if cached is not None and cached[1] == current_mtime:
|
||||
return cached[0]
|
||||
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, current_mtime)
|
||||
|
||||
return memory_data
|
||||
|
||||
def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
"""Reload memory data from file, forcing cache invalidation."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
memory_data = self._load_from_file(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime if file_path.exists() else None
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
return memory_data
|
||||
|
||||
def save(self, memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> bool:
|
||||
"""Save memory data to file and update cache (atomic write)."""
|
||||
file_path = self._get_memory_file_path(agent_name, user_id=user_id)
|
||||
cache_key = self._cache_key(agent_name, user_id=user_id)
|
||||
|
||||
try:
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
memory_data = {**memory_data, "lastUpdated": utc_now_iso_z()}
|
||||
|
||||
temp_path = file_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
|
||||
with open(temp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(memory_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
temp_path.replace(file_path)
|
||||
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
|
||||
with self._cache_lock:
|
||||
self._memory_cache[cache_key] = (memory_data, mtime)
|
||||
logger.info("Memory saved to %s", file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error("Failed to save memory file: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
_storage_instance: FileMemoryStorage | None = None
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_memory_storage() -> FileMemoryStorage:
|
||||
"""Get the global FileMemoryStorage singleton."""
|
||||
global _storage_instance
|
||||
if _storage_instance is not None:
|
||||
return _storage_instance
|
||||
with _storage_lock:
|
||||
if _storage_instance is None:
|
||||
_storage_instance = FileMemoryStorage()
|
||||
return _storage_instance
|
||||
|
||||
|
||||
__all__ = [
|
||||
"create_empty_memory",
|
||||
"utc_now_iso_z",
|
||||
"FileMemoryStorage",
|
||||
"get_memory_storage",
|
||||
]
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""Hooks fired before summarization removes messages from state(原版 DeerFlow)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.agents.memory.message_processing import detect_correction, detect_reinforcement, filter_messages_for_memory
|
||||
from deerflow.agents.memory.v1.queue import get_memory_queue
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
|
||||
|
||||
def memory_flush_hook(event: object) -> None:
|
||||
"""Flush messages about to be summarized into the V1 memory queue.
|
||||
|
||||
``event`` is expected to be a ``SummarizationEvent``-like object with:
|
||||
- ``thread_id: str``
|
||||
- ``messages_to_summarize: list``
|
||||
- ``agent_name: str | None``
|
||||
- ``runtime: RunnableConfig``
|
||||
"""
|
||||
if not get_memory_config().enabled:
|
||||
return
|
||||
|
||||
thread_id = getattr(event, "thread_id", None)
|
||||
if not thread_id:
|
||||
return
|
||||
|
||||
messages_to_summarize = getattr(event, "messages_to_summarize", [])
|
||||
filtered_messages = filter_messages_for_memory(list(messages_to_summarize))
|
||||
user_messages = [m for m in filtered_messages if getattr(m, "type", None) == "human"]
|
||||
assistant_messages = [m for m in filtered_messages if getattr(m, "type", None) == "ai"]
|
||||
if not user_messages or not assistant_messages:
|
||||
return
|
||||
|
||||
correction_detected = detect_correction(filtered_messages)
|
||||
reinforcement_detected = not correction_detected and detect_reinforcement(filtered_messages)
|
||||
|
||||
try:
|
||||
from deerflow.runtime.user_context import resolve_runtime_user_id
|
||||
runtime = getattr(event, "runtime", None)
|
||||
user_id = resolve_runtime_user_id(runtime) if runtime is not None else None
|
||||
except Exception: # noqa: BLE001
|
||||
user_id = None
|
||||
|
||||
agent_name = getattr(event, "agent_name", None)
|
||||
queue = get_memory_queue()
|
||||
queue.add_nowait(
|
||||
thread_id=thread_id,
|
||||
messages=filtered_messages,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["memory_flush_hook"]
|
||||
+495
@@ -0,0 +1,495 @@
|
||||
"""Memory V1 — updater(原版 DeerFlow MemoryUpdater)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.v1.prompt import (
|
||||
MEMORY_UPDATE_PROMPT,
|
||||
format_conversation_for_update,
|
||||
)
|
||||
from deerflow.agents.memory.v1.storage import (
|
||||
create_empty_memory,
|
||||
get_memory_storage,
|
||||
utc_now_iso_z,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SYNC_MEMORY_UPDATER_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=4,
|
||||
thread_name_prefix="v1-memory-updater-sync",
|
||||
)
|
||||
atexit.register(lambda: _SYNC_MEMORY_UPDATER_EXECUTOR.shutdown(wait=False))
|
||||
|
||||
|
||||
def _get_v1_cfg():
|
||||
"""返回 V1MemoryConfig(含 max_facts, fact_confidence_threshold, model_name)。"""
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
return get_memory_config().v1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 向后兼容的便捷函数(供 gateway router 等调用)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def reload_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
return get_memory_storage().reload(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def import_memory_data(memory_data: dict[str, Any], agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
storage = get_memory_storage()
|
||||
if not storage.save(memory_data, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save imported memory data")
|
||||
return storage.load(agent_name, user_id=user_id)
|
||||
|
||||
|
||||
def clear_memory_data(agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
cleared = create_empty_memory()
|
||||
if not get_memory_storage().save(cleared, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save cleared memory data")
|
||||
return cleared
|
||||
|
||||
|
||||
def _validate_confidence(confidence: float) -> float:
|
||||
if not math.isfinite(confidence) or confidence < 0 or confidence > 1:
|
||||
raise ValueError("confidence")
|
||||
return confidence
|
||||
|
||||
|
||||
def create_memory_fact(
|
||||
content: str,
|
||||
category: str = "context",
|
||||
confidence: float = 0.5,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_content = content.strip()
|
||||
if not normalized_content:
|
||||
raise ValueError("content")
|
||||
normalized_category = category.strip() or "context"
|
||||
validated_confidence = _validate_confidence(confidence)
|
||||
now = utc_now_iso_z()
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
facts = list(memory_data.get("facts", []))
|
||||
facts.append({
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": normalized_category,
|
||||
"confidence": validated_confidence,
|
||||
"createdAt": now,
|
||||
"source": "manual",
|
||||
})
|
||||
updated_memory["facts"] = facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError("Failed to save memory data after creating fact")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def delete_memory_fact(fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
facts = memory_data.get("facts", [])
|
||||
updated_facts = [f for f in facts if f.get("id") != fact_id]
|
||||
if len(updated_facts) == len(facts):
|
||||
raise KeyError(fact_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after deleting fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
def update_memory_fact(
|
||||
fact_id: str,
|
||||
content: str | None = None,
|
||||
category: str | None = None,
|
||||
confidence: float | None = None,
|
||||
agent_name: str | None = None,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
memory_data = get_memory_data(agent_name, user_id=user_id)
|
||||
updated_memory = dict(memory_data)
|
||||
updated_facts: list[dict[str, Any]] = []
|
||||
found = False
|
||||
for fact in memory_data.get("facts", []):
|
||||
if fact.get("id") == fact_id:
|
||||
found = True
|
||||
updated_fact = dict(fact)
|
||||
if content is not None:
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
raise ValueError("content")
|
||||
updated_fact["content"] = normalized
|
||||
if category is not None:
|
||||
updated_fact["category"] = category.strip() or "context"
|
||||
if confidence is not None:
|
||||
updated_fact["confidence"] = _validate_confidence(confidence)
|
||||
updated_facts.append(updated_fact)
|
||||
else:
|
||||
updated_facts.append(fact)
|
||||
if not found:
|
||||
raise KeyError(fact_id)
|
||||
updated_memory["facts"] = updated_facts
|
||||
if not get_memory_storage().save(updated_memory, agent_name, user_id=user_id):
|
||||
raise OSError(f"Failed to save memory data after updating fact '{fact_id}'")
|
||||
return updated_memory
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内容提取辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
"""从 LLM response content(str 或 list of blocks)提取纯文本。"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
pieces: list[str] = []
|
||||
pending: list[str] = []
|
||||
|
||||
def flush():
|
||||
if pending:
|
||||
pieces.append("".join(pending))
|
||||
pending.clear()
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
pending.append(block)
|
||||
elif isinstance(block, dict):
|
||||
flush()
|
||||
text_val = block.get("text")
|
||||
if isinstance(text_val, str):
|
||||
pieces.append(text_val)
|
||||
flush()
|
||||
return "\n".join(pieces)
|
||||
return str(content)
|
||||
|
||||
|
||||
_UPLOAD_SENTENCE_RE = re.compile(
|
||||
r"[^.!?]*\b(?:"
|
||||
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
|
||||
r"|file\s+upload"
|
||||
r"|/mnt/user-data/uploads/"
|
||||
r"|<uploaded_files>"
|
||||
r")[^.!?]*[.!?]?\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _strip_upload_mentions_from_memory(memory_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""移除记忆中关于文件上传事件的句子(上传文件是会话级临时的)。"""
|
||||
for section in ("user", "history"):
|
||||
section_data = memory_data.get(section, {})
|
||||
for _key, val in section_data.items():
|
||||
if isinstance(val, dict) and "summary" in val:
|
||||
cleaned = _UPLOAD_SENTENCE_RE.sub("", val["summary"]).strip()
|
||||
cleaned = re.sub(r" +", " ", cleaned)
|
||||
val["summary"] = cleaned
|
||||
|
||||
facts = memory_data.get("facts", [])
|
||||
if facts:
|
||||
memory_data["facts"] = [f for f in facts if not _UPLOAD_SENTENCE_RE.search(f.get("content", ""))]
|
||||
return memory_data
|
||||
|
||||
|
||||
def _fact_content_key(content: Any) -> str | None:
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
stripped = content.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
return stripped.casefold()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MemoryUpdater(原版核心类)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class MemoryUpdater:
|
||||
"""Updates memory using LLM based on conversation context."""
|
||||
|
||||
def __init__(self, model_name: str | None = None) -> None:
|
||||
self._model_name = model_name
|
||||
|
||||
def _get_model(self):
|
||||
from deerflow.models import create_chat_model
|
||||
cfg = _get_v1_cfg()
|
||||
model_name = self._model_name or cfg.model_name
|
||||
return create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
|
||||
def _build_correction_hint(self, correction_detected: bool, reinforcement_detected: bool) -> str:
|
||||
correction_hint = ""
|
||||
if correction_detected:
|
||||
correction_hint = (
|
||||
"IMPORTANT: Explicit correction signals were detected in this conversation. "
|
||||
"Pay special attention to what the agent got wrong, what the user corrected, "
|
||||
"and record the correct approach as a fact with category "
|
||||
'"correction" and confidence >= 0.95 when appropriate.'
|
||||
)
|
||||
if reinforcement_detected:
|
||||
reinforcement_hint = (
|
||||
"IMPORTANT: Positive reinforcement signals were detected in this conversation. "
|
||||
"The user explicitly confirmed the agent's approach was correct or helpful. "
|
||||
"Record the confirmed approach, style, or preference as a fact with category "
|
||||
'"preference" or "behavior" and confidence >= 0.9 when appropriate.'
|
||||
)
|
||||
correction_hint = (correction_hint + "\n" + reinforcement_hint).strip() if correction_hint else reinforcement_hint
|
||||
return correction_hint
|
||||
|
||||
def _prepare_update_prompt(
|
||||
self,
|
||||
messages: list[Any],
|
||||
agent_name: str | None,
|
||||
correction_detected: bool,
|
||||
reinforcement_detected: bool,
|
||||
user_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], str] | None:
|
||||
from deerflow.config.memory_config import get_memory_config
|
||||
config = get_memory_config()
|
||||
if not config.enabled or not messages:
|
||||
return None
|
||||
|
||||
current_memory = get_memory_data(agent_name, user_id=user_id)
|
||||
conversation_text = format_conversation_for_update(messages)
|
||||
if not conversation_text.strip():
|
||||
return None
|
||||
|
||||
correction_hint = self._build_correction_hint(correction_detected, reinforcement_detected)
|
||||
prompt = MEMORY_UPDATE_PROMPT.format(
|
||||
current_memory=json.dumps(current_memory, indent=2, ensure_ascii=False),
|
||||
conversation=conversation_text,
|
||||
correction_hint=correction_hint,
|
||||
)
|
||||
return current_memory, prompt
|
||||
|
||||
def _finalize_update(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
response_content: Any,
|
||||
thread_id: str | None,
|
||||
agent_name: str | None,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
response_text = _extract_text(response_content).strip()
|
||||
if response_text.startswith("```"):
|
||||
lines = response_text.split("\n")
|
||||
response_text = "\n".join(lines[1:-1] if lines[-1] == "```" else lines[1:])
|
||||
|
||||
update_data = json.loads(response_text)
|
||||
updated_memory = self._apply_updates(copy.deepcopy(current_memory), update_data, thread_id)
|
||||
updated_memory = _strip_upload_mentions_from_memory(updated_memory)
|
||||
return get_memory_storage().save(updated_memory, agent_name, user_id=user_id)
|
||||
|
||||
async def aupdate_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
return await asyncio.to_thread(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _do_update_memory_sync(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
prepared = self._prepare_update_prompt(
|
||||
messages=messages,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
if prepared is None:
|
||||
return False
|
||||
|
||||
current_memory, prompt = prepared
|
||||
model = self._get_model()
|
||||
response = model.invoke(prompt, config={"run_name": "v1_memory_agent"})
|
||||
return self._finalize_update(
|
||||
current_memory=current_memory,
|
||||
response_content=response.content,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
user_id=user_id,
|
||||
)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("V1 memory: failed to parse LLM response: %s", e)
|
||||
return False
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("V1 memory update failed: %s", e)
|
||||
return False
|
||||
|
||||
def update_memory(
|
||||
self,
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop is not None and loop.is_running():
|
||||
try:
|
||||
future = _SYNC_MEMORY_UPDATER_EXECUTOR.submit(
|
||||
self._do_update_memory_sync,
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
return future.result()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("V1 memory: failed to offload update to executor")
|
||||
return False
|
||||
|
||||
return self._do_update_memory_sync(
|
||||
messages=messages,
|
||||
thread_id=thread_id,
|
||||
agent_name=agent_name,
|
||||
correction_detected=correction_detected,
|
||||
reinforcement_detected=reinforcement_detected,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
def _apply_updates(
|
||||
self,
|
||||
current_memory: dict[str, Any],
|
||||
update_data: dict[str, Any],
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = _get_v1_cfg()
|
||||
now = utc_now_iso_z()
|
||||
|
||||
# 更新 user context 段
|
||||
user_updates = update_data.get("user", {})
|
||||
for section in ("workContext", "personalContext", "topOfMind"):
|
||||
section_data = user_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("user", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 更新 history 段
|
||||
history_updates = update_data.get("history", {})
|
||||
for section in ("recentMonths", "earlierContext", "longTermBackground"):
|
||||
section_data = history_updates.get(section, {})
|
||||
if section_data.get("shouldUpdate") and section_data.get("summary"):
|
||||
current_memory.setdefault("history", {})[section] = {
|
||||
"summary": section_data["summary"],
|
||||
"updatedAt": now,
|
||||
}
|
||||
|
||||
# 删除事实
|
||||
facts_to_remove = set(update_data.get("factsToRemove", []))
|
||||
if facts_to_remove:
|
||||
current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove]
|
||||
|
||||
# 新增事实(去重 + 置信度过滤)
|
||||
existing_keys = {
|
||||
key for key in (_fact_content_key(f.get("content")) for f in current_memory.get("facts", []))
|
||||
if key is not None
|
||||
}
|
||||
|
||||
for fact in update_data.get("newFacts", []):
|
||||
confidence = fact.get("confidence", 0.5)
|
||||
if confidence < cfg.fact_confidence_threshold:
|
||||
continue
|
||||
raw_content = fact.get("content", "")
|
||||
if not isinstance(raw_content, str):
|
||||
continue
|
||||
normalized_content = raw_content.strip()
|
||||
fact_key = _fact_content_key(normalized_content)
|
||||
if fact_key is not None and fact_key in existing_keys:
|
||||
continue
|
||||
|
||||
fact_entry: dict[str, Any] = {
|
||||
"id": f"fact_{uuid.uuid4().hex[:8]}",
|
||||
"content": normalized_content,
|
||||
"category": fact.get("category", "context"),
|
||||
"confidence": confidence,
|
||||
"createdAt": now,
|
||||
"source": thread_id or "unknown",
|
||||
}
|
||||
source_error = fact.get("sourceError")
|
||||
if isinstance(source_error, str) and source_error.strip():
|
||||
fact_entry["sourceError"] = source_error.strip()
|
||||
|
||||
current_memory["facts"].append(fact_entry)
|
||||
if fact_key is not None:
|
||||
existing_keys.add(fact_key)
|
||||
|
||||
# 超出上限时驱逐置信度最低的事实
|
||||
if len(current_memory["facts"]) > cfg.max_facts:
|
||||
current_memory["facts"] = sorted(
|
||||
current_memory["facts"],
|
||||
key=lambda f: f.get("confidence", 0),
|
||||
reverse=True,
|
||||
)[:cfg.max_facts]
|
||||
|
||||
return current_memory
|
||||
|
||||
|
||||
def update_memory_from_conversation(
|
||||
messages: list[Any],
|
||||
thread_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
correction_detected: bool = False,
|
||||
reinforcement_detected: bool = False,
|
||||
user_id: str | None = None,
|
||||
) -> bool:
|
||||
updater = MemoryUpdater()
|
||||
return updater.update_memory(messages, thread_id, agent_name, correction_detected, reinforcement_detected, user_id=user_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryUpdater",
|
||||
"update_memory_from_conversation",
|
||||
"get_memory_data",
|
||||
"reload_memory_data",
|
||||
"import_memory_data",
|
||||
"clear_memory_data",
|
||||
"create_memory_fact",
|
||||
"delete_memory_fact",
|
||||
"update_memory_fact",
|
||||
]
|
||||
+7
-7
@@ -171,21 +171,21 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
|
||||
|
||||
def _build_retry_message(self, attempt: int, wait_ms: int, reason: str) -> str:
|
||||
seconds = max(1, round(wait_ms / 1000))
|
||||
reason_text = "provider is busy" if reason == "busy" else "provider request failed temporarily"
|
||||
return f"LLM request retry {attempt}/{self.retry_max_attempts}: {reason_text}. Retrying in {seconds}s."
|
||||
reason_text = "服务繁忙" if reason == "busy" else "网络/连接异常"
|
||||
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
|
||||
|
||||
def _build_circuit_breaker_message(self) -> str:
|
||||
return "The configured LLM provider is currently unavailable due to continuous failures. Circuit breaker is engaged to protect the system. Please wait a moment before trying again."
|
||||
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
|
||||
|
||||
def _build_user_message(self, exc: BaseException, reason: str) -> str:
|
||||
detail = _extract_error_detail(exc)
|
||||
if reason == "quota":
|
||||
return "The configured LLM provider rejected the request because the account is out of quota, billing is unavailable, or usage is restricted. Please fix the provider account and try again."
|
||||
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
|
||||
if reason == "auth":
|
||||
return "The configured LLM provider rejected the request because authentication or access is invalid. Please check the provider credentials and try again."
|
||||
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
|
||||
if reason in {"busy", "transient"}:
|
||||
return "The configured LLM provider is temporarily unavailable after multiple retries. Please wait a moment and continue the conversation."
|
||||
return f"LLM request failed: {detail}"
|
||||
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
|
||||
return f"大模型因「{detail}」原因调用失败。"
|
||||
|
||||
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
|
||||
try:
|
||||
|
||||
+11
-4
@@ -90,16 +90,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
|
||||
state_schema = MemoryMiddlewareState
|
||||
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None):
|
||||
def __init__(self, agent_name: str | None = None, *, memory_config: "MemoryConfig | None" = None, injection_enabled: bool = True):
|
||||
"""初始化 MemoryMiddleware。
|
||||
|
||||
Args:
|
||||
agent_name: 提供时记忆按 agent 隔离;None 用默认 agent。
|
||||
memory_config: 显式记忆配置;省略时用全局配置。
|
||||
injection_enabled: 运行时开关;``False`` 时跳过 Hindsight 动态召回注入。
|
||||
"""
|
||||
super().__init__()
|
||||
self._agent_name = agent_name
|
||||
self._memory_config = memory_config
|
||||
self._injection_enabled = injection_enabled
|
||||
|
||||
# ---- 运行时上下文解析 -------------------------------------------------
|
||||
|
||||
@@ -115,15 +117,18 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
def _resolve_agent_id(self, runtime: Runtime | None) -> str:
|
||||
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。"""
|
||||
def _resolve_agent_id(self, runtime: Runtime | None) -> str | None:
|
||||
"""解析 agent_id;优先 middleware 构造时的 agent_name,再 runtime context。
|
||||
|
||||
无 agent 上下文时返回 None,使 V1 记忆存入全局文件(users/{id}/memory.json)。
|
||||
"""
|
||||
if self._agent_name:
|
||||
return self._agent_name
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
agent_id = runtime.context.get("agent_id")
|
||||
if agent_id:
|
||||
return str(agent_id)
|
||||
return "default"
|
||||
return None
|
||||
|
||||
# ---- wrap_model_call:Hindsight 动态召回注入 -------------------------
|
||||
|
||||
@@ -133,6 +138,8 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
|
||||
召回结果包在 ``<memory-context>`` 标签里并带系统提示,告诉模型这是
|
||||
背景信息而非用户的新输入。
|
||||
"""
|
||||
if not self._injection_enabled:
|
||||
return None
|
||||
config = self._memory_config or get_memory_config()
|
||||
if not config.enabled:
|
||||
return None
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Middleware for intercepting skill calls and stopping execution when specific skills are invoked."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command, interrupt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_name_task(text: str) -> tuple[str, str]:
|
||||
"""Extract `agent_name` and `task_content` from a bash invocation that triggers
|
||||
a sub-agent via ``python3 scripts/agent_orchestration.py "<agent>" "<task>"``.
|
||||
|
||||
Returns empty strings when no match is found so the caller can treat the
|
||||
absence of a match as "not a target skill call".
|
||||
"""
|
||||
try:
|
||||
pattern = r'\.py\s+"([^"]*)"\s+"([^"]*)"'
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
return match.group(1), match.group(2)
|
||||
return "", ""
|
||||
except Exception:
|
||||
return "", ""
|
||||
|
||||
|
||||
class SkillStopMiddlewareState(AgentState):
|
||||
"""Compatible with the `ThreadState` schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SkillStopMiddleware(AgentMiddleware[SkillStopMiddlewareState]):
|
||||
"""Intercepts skill file reads and stops execution when specific skills are invoked.
|
||||
|
||||
When the model reads a file from a skill directory that matches one of the
|
||||
configured skill names, this middleware:
|
||||
1. Intercepts the tool call before execution
|
||||
2. Returns a Command that stops execution (goes to END)
|
||||
3. Optionally returns a message indicating which skill was triggered
|
||||
|
||||
This is useful for implementing "skill triggers" that pause the agent
|
||||
when certain skills are about to be used.
|
||||
"""
|
||||
|
||||
state_schema = SkillStopMiddlewareState
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
skill_names: list[str],
|
||||
skills_root: str = "/mnt/skills",
|
||||
stop_message: str | None = None,
|
||||
stop_on_match: bool = True,
|
||||
):
|
||||
"""Initialize the skill stop middleware.
|
||||
|
||||
Args:
|
||||
skill_names: List of skill names to intercept (e.g., ["newsletter-generation", "podcast-generation"]).
|
||||
skills_root: Root directory where skills are located (default: "/mnt/skills").
|
||||
stop_message: Optional message to return when stopping. If None, a default message is used.
|
||||
stop_on_match: If True, stop when skill matches. If False, continue execution (for logging only).
|
||||
"""
|
||||
super().__init__()
|
||||
self._skill_names = set(skill_names)
|
||||
self._skills_root = skills_root.rstrip("/")
|
||||
self._stop_message = stop_message
|
||||
self._stop_on_match = stop_on_match
|
||||
|
||||
def _is_target_skill_call(self, tool_call: dict[str, Any]) -> str | None:
|
||||
"""Return the matched skill name if this tool call invokes one of the
|
||||
configured target skills, otherwise ``None``.
|
||||
|
||||
Two invocation patterns are recognized:
|
||||
|
||||
1. **Structured tool call** — the model emits a tool call whose ``name``
|
||||
literally matches one of the configured skill names (e.g. the
|
||||
multi-agent leader calling
|
||||
``agent_orchestration({"agent_name": "...", "task": "..."})``).
|
||||
This is the natural form most LLMs produce when told to "use a
|
||||
skill", and it's the form that drives the roundtable dispatch.
|
||||
|
||||
2. **Bash + argv** — the legacy form where the lead agent reaches into
|
||||
a skill by running ``python3 scripts/<skill-name>.py "<arg1>" "<arg2>"``.
|
||||
Kept for backward compatibility with skills that use bash for
|
||||
dispatch.
|
||||
"""
|
||||
name = tool_call.get("name") or ""
|
||||
lowered_targets = {s.lower() for s in self._skill_names}
|
||||
|
||||
# Pattern 1: structured tool call whose tool name IS the skill name.
|
||||
if name and (name in self._skill_names or name.lower() in lowered_targets):
|
||||
logger.info("SkillStopMiddleware: matched structured skill call name=%s", name)
|
||||
return name
|
||||
|
||||
# Pattern 2: bash invocation with argv-style skill name.
|
||||
if name == "bash" and "args" in tool_call:
|
||||
skill_name, _ = extract_name_task(str(tool_call["args"]))
|
||||
if skill_name and (skill_name in self._skill_names or skill_name.lower() in lowered_targets):
|
||||
logger.info("SkillStopMiddleware: matched bash skill call name=%s", skill_name)
|
||||
return skill_name
|
||||
|
||||
return None
|
||||
|
||||
def _handle_skill_stop(self, request: ToolCallRequest, skill_name: str) -> Command:
|
||||
"""Build the stop response — an interrupt that includes a ToolMessage
|
||||
describing which skill was triggered."""
|
||||
logger.info("Skill stop triggered: skill_name=%s", skill_name)
|
||||
|
||||
tool_call_id = request.tool_call.get("id", "")
|
||||
message_content = self._stop_message or f"⏹️ Skill '{skill_name}' triggered. Execution stopped."
|
||||
|
||||
tool_message = ToolMessage(
|
||||
id=f"skill_stop:{tool_call_id}",
|
||||
content=message_content,
|
||||
tool_call_id=tool_call_id,
|
||||
name="skill_stop",
|
||||
)
|
||||
|
||||
return interrupt(
|
||||
{
|
||||
"messages": [tool_message],
|
||||
"skill_name": skill_name,
|
||||
}
|
||||
)
|
||||
|
||||
@override
|
||||
def wrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
matched_skill = self._is_target_skill_call(request.tool_call)
|
||||
if matched_skill:
|
||||
if self._stop_on_match:
|
||||
return self._handle_skill_stop(request, matched_skill)
|
||||
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
matched_skill = self._is_target_skill_call(request.tool_call)
|
||||
if matched_skill:
|
||||
if self._stop_on_match:
|
||||
return self._handle_skill_stop(request, matched_skill)
|
||||
logger.info("Skill detected (not stopping): skill_name=%s", matched_skill)
|
||||
return await handler(request)
|
||||
+41
-2
@@ -105,8 +105,16 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
return prompt, user_msg
|
||||
|
||||
def _strip_think_tags(self, text: str) -> str:
|
||||
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, DeepSeek-R1)."""
|
||||
return re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE).strip()
|
||||
"""Remove <think>...</think> blocks emitted by reasoning models (e.g. minimax, GLM, DeepSeek-R1).
|
||||
|
||||
Also handles malformed cases: an unclosed `<think>` (response truncated mid-reasoning)
|
||||
or a stray `</think>` (open tag eaten by upstream parsing) — both can otherwise leak
|
||||
raw reasoning into the title.
|
||||
"""
|
||||
text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"<think>[\s\S]*$", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"^[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
|
||||
return text.strip()
|
||||
|
||||
def _parse_title(self, content: object) -> str:
|
||||
"""Normalize model output into a clean title string."""
|
||||
@@ -146,6 +154,36 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
_, user_msg = self._build_title_prompt(state)
|
||||
return {"title": self._fallback_title(user_msg)}
|
||||
|
||||
def _force_disable_thinking(self, model: object) -> None:
|
||||
"""Force-disable thinking on the title model via `extra_body` overrides.
|
||||
|
||||
Title generation never needs reasoning. The model factory only emits
|
||||
disable params when the model config declares `when_thinking_disabled`
|
||||
or a matching `when_thinking_enabled` shape — many user-added reasoning
|
||||
models (e.g. GLM-4.5/5) only set `supports_thinking: true` and would
|
||||
otherwise keep thinking on, which is slow and leaks <think> tags into
|
||||
the title prompt response. Injecting both common OpenAI-compatible
|
||||
disable shapes covers Zhipu GLM (`thinking.type`) and Qwen/vLLM
|
||||
(`chat_template_kwargs.enable_thinking`); other providers ignore
|
||||
unknown body fields.
|
||||
"""
|
||||
if not hasattr(model, "extra_body"):
|
||||
return
|
||||
current = getattr(model, "extra_body", None) or {}
|
||||
merged_chat_template_kwargs = {
|
||||
**(current.get("chat_template_kwargs") or {}),
|
||||
"enable_thinking": False,
|
||||
}
|
||||
merged = {
|
||||
**current,
|
||||
"thinking": {**(current.get("thinking") or {}), "type": "disabled"},
|
||||
"chat_template_kwargs": merged_chat_template_kwargs,
|
||||
}
|
||||
try:
|
||||
model.extra_body = merged
|
||||
except Exception:
|
||||
logger.debug("Could not override extra_body to disable thinking on title model", exc_info=True)
|
||||
|
||||
async def _agenerate_title_result(self, state: TitleMiddlewareState) -> dict | None:
|
||||
"""Generate a title asynchronously and fall back locally on failure."""
|
||||
if not self._should_generate_title(state):
|
||||
@@ -162,6 +200,7 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
model = create_chat_model(name=config.model_name, **model_kwargs)
|
||||
else:
|
||||
model = create_chat_model(**model_kwargs)
|
||||
self._force_disable_thinking(model)
|
||||
response = await model.ainvoke(prompt, config=self._get_runnable_config())
|
||||
title = self._parse_title(response.content)
|
||||
if title:
|
||||
|
||||
@@ -22,6 +22,7 @@ from deerflow.config.sandbox_config import SandboxConfig
|
||||
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
|
||||
from deerflow.config.skill_evolution_runtime import get_overrides_mtime as _get_skill_evolution_overrides_mtime
|
||||
from deerflow.config.skill_evolution_runtime import load_overrides as _load_skill_evolution_overrides
|
||||
from deerflow.config.skill_stop_config import SkillStopConfig
|
||||
from deerflow.config.skills_config import SkillsConfig
|
||||
from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict
|
||||
from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict
|
||||
@@ -117,6 +118,8 @@ class AppConfig(BaseModel):
|
||||
subagents: SubagentsAppConfig = Field(default_factory=SubagentsAppConfig, description="Subagent runtime configuration")
|
||||
guardrails: GuardrailsConfig = Field(default_factory=GuardrailsConfig, description="Guardrail middleware configuration")
|
||||
circuit_breaker: CircuitBreakerConfig = Field(default_factory=CircuitBreakerConfig, description="LLM circuit breaker configuration")
|
||||
excluded_tools: list[str] = Field(default_factory=list, description="Tool names to exclude from the main agent (e.g. ['task', 'ask_clarification'])")
|
||||
skill_stop: SkillStopConfig = Field(default_factory=SkillStopConfig, description="Skill stop middleware configuration")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
database: DatabaseConfig = Field(default_factory=DatabaseConfig, description="Unified database backend configuration")
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
|
||||
@@ -113,15 +113,30 @@ class SecurityConfig(BaseModel):
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class V1MemoryConfig(BaseModel):
|
||||
"""原版 DeerFlow V1 记忆 provider 配置 (memory.json + LLM 自动提取)。"""
|
||||
|
||||
max_facts: int = Field(default=100, ge=10, le=500, description="最多保留的事实条数")
|
||||
fact_confidence_threshold: float = Field(default=0.7, ge=0.0, le=1.0, description="事实置信度阈值,低于此值不保存")
|
||||
debounce_seconds: int = Field(default=30, ge=5, le=300, description="对话结束后延迟多少秒才触发 LLM 提取")
|
||||
model_name: str = Field(default="", description="用于事实提取的 LLM 模型名;空则继承全局默认模型")
|
||||
token_budget: int = Field(default=2000, ge=200, le=8000, description="注入系统提示词的 token 预算")
|
||||
|
||||
|
||||
class MemoryConfig(BaseModel):
|
||||
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="记忆子系统总开关")
|
||||
version: str = Field(
|
||||
default="v2",
|
||||
description="记忆系统版本:v1(原版 DeerFlow,memory.json + LLM 自动提取) 或 v2(USER.md/MEMORY.md)",
|
||||
)
|
||||
provider: str = Field(
|
||||
default="builtin",
|
||||
description="启用的外部 provider 名:builtin(无外部) 或 hindsight",
|
||||
)
|
||||
|
||||
v1: V1MemoryConfig = Field(default_factory=V1MemoryConfig)
|
||||
builtin: BuiltinMemoryConfig = Field(default_factory=BuiltinMemoryConfig)
|
||||
hindsight: HindsightMemoryConfig = Field(default_factory=HindsightMemoryConfig)
|
||||
injection: InjectionConfig = Field(default_factory=InjectionConfig)
|
||||
@@ -288,6 +303,7 @@ async def delete_user_memory_override_db(user_id: str, field: str | None, store:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"V1MemoryConfig",
|
||||
"BuiltinMemoryConfig",
|
||||
"HindsightMemoryConfig",
|
||||
"InjectionConfig",
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Configuration for skill stop middleware."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SkillStopConfig(BaseModel):
|
||||
"""Configuration for the skill stop middleware.
|
||||
|
||||
When enabled, the middleware will intercept skill invocations and stop
|
||||
execution when a matching skill is about to be used.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable skill stop middleware. When True, execution will stop when specified skills are invoked.",
|
||||
)
|
||||
|
||||
skill_names: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="List of skill names to intercept. When the model reads a file from these skill directories, execution will stop.",
|
||||
)
|
||||
|
||||
stop_message: str | None = Field(
|
||||
default=None,
|
||||
description="Custom message to display when stopping. If None, a default message is used.",
|
||||
)
|
||||
|
||||
stop_on_match: bool = Field(
|
||||
default=True,
|
||||
description="If True, stop execution when skill matches. If False, only log the match without stopping.",
|
||||
)
|
||||
@@ -6,6 +6,18 @@ from pydantic import BaseModel, Field
|
||||
from deerflow.config.runtime_paths import project_root, resolve_path
|
||||
|
||||
|
||||
class SecurityScanConfig(BaseModel):
|
||||
"""Toggle for the LLM-based skill security scanner."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"When false, skip the LLM security scan applied to skill installs, "
|
||||
"custom-skill edits, and rollbacks. The scan is then treated as 'allow'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SkillsConfig(BaseModel):
|
||||
"""Configuration for skills system"""
|
||||
|
||||
@@ -21,6 +33,10 @@ class SkillsConfig(BaseModel):
|
||||
default="/mnt/skills",
|
||||
description="Path where skills are mounted in the sandbox container",
|
||||
)
|
||||
security_scan: SecurityScanConfig = Field(
|
||||
default_factory=SecurityScanConfig,
|
||||
description="LLM security-scan toggle for skill content.",
|
||||
)
|
||||
|
||||
def get_skills_path(self) -> Path:
|
||||
"""
|
||||
|
||||
@@ -11,13 +11,16 @@ None and fall back to in-memory implementations.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
@@ -77,6 +80,10 @@ async def _run_alembic_upgrade(url: str) -> None:
|
||||
operable on the ``create_all`` schema; the operator just won't get
|
||||
in-place fixes until they investigate.
|
||||
"""
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in ("1", "true", "yes"):
|
||||
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations (schema stays at create_all)")
|
||||
return
|
||||
|
||||
ini_path = Path(__file__).parent / "migrations" / "alembic.ini"
|
||||
if not ini_path.exists():
|
||||
logger.warning("Alembic config not found at %s; skipping auto-migrate", ini_path)
|
||||
@@ -102,14 +109,31 @@ async def _run_alembic_upgrade(url: str) -> None:
|
||||
# at the directory of alembic.ini itself — already correct for us.
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
# Bound the upgrade so a stuck DDL (e.g. MySQL metadata-lock wait) cannot
|
||||
# hang startup forever. Tunable via DEER_FLOW_ALEMBIC_TIMEOUT (seconds).
|
||||
# NOTE: asyncio.wait_for cancels the *await* on timeout; the worker thread
|
||||
# itself keeps running (Python can't kill threads). That's acceptable —
|
||||
# the orphan thread will exit when its MySQL connection eventually errors
|
||||
# or when the process restarts. The point here is to unblock startup.
|
||||
try:
|
||||
await asyncio.to_thread(_do_upgrade)
|
||||
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
|
||||
except ValueError:
|
||||
timeout_s = 60.0
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s)
|
||||
logger.info("Alembic migrations applied (or already up-to-date)")
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(
|
||||
"Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.",
|
||||
timeout_s,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
@@ -132,11 +156,50 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def init_engine(
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def _auto_create_mysql_db(url: str) -> None:
|
||||
"""Create the target MySQL database when it is missing.
|
||||
|
||||
``create_all`` can create tables, but MySQL requires the database/schema to
|
||||
exist before the application URL can connect. We connect to the built-in
|
||||
``mysql`` maintenance database on the same server and issue a conservative
|
||||
``CREATE DATABASE IF NOT EXISTS``.
|
||||
"""
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
|
||||
|
||||
maint_url = parsed.set(database="mysql")
|
||||
maint_engine = create_async_engine(
|
||||
maint_url.render_as_string(hide_password=False),
|
||||
isolation_level="AUTOCOMMIT",
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
quoted = "`" + db_name.replace("`", "``") + "`"
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
f"CREATE DATABASE IF NOT EXISTS {quoted} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||||
)
|
||||
)
|
||||
logger.info("Auto-created MySQL database if missing: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
url: str = "",
|
||||
@@ -239,25 +302,41 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
await _run_alembic_upgrade(url)
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s", backend)
|
||||
|
||||
|
||||
+23
-7
@@ -9,9 +9,11 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -44,13 +46,27 @@ def run_migrations_offline() -> None:
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
|
||||
# On MySQL, shrink the session lock waits so a blocked DDL fails fast
|
||||
# instead of hanging on the server default (lock_wait_timeout = 31536000s
|
||||
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
|
||||
# transaction on another connection can freeze startup indefinitely.
|
||||
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
|
||||
if connection.dialect.name == "mysql":
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""Merge agent pin/featured and user memory config migration heads.
|
||||
|
||||
Revision ID: 20260521_02
|
||||
Revises: 20260520_03, 20260521_01
|
||||
Create Date: 2026-05-21
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
revision: str = "20260521_02"
|
||||
down_revision: str | tuple[str, str] = ("20260520_03", "20260521_01")
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""Add description column to scheduled_tasks.
|
||||
|
||||
Revision ID: 20260522_01
|
||||
Revises: 20260521_02
|
||||
Create Date: 2026-05-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260522_01"
|
||||
down_revision = "20260521_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_tasks", "description"):
|
||||
op.add_column(
|
||||
"scheduled_tasks",
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("scheduled_tasks", "description"):
|
||||
op.drop_column("scheduled_tasks", "description")
|
||||
+1
@@ -31,6 +31,7 @@ class ScheduledTaskRow(Base):
|
||||
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
prompt: Mapped[str] = mapped_column(Text)
|
||||
schedule_text: Mapped[str] = mapped_column(String(256))
|
||||
cron_expr: Mapped[str] = mapped_column(String(128))
|
||||
|
||||
+1
@@ -76,6 +76,7 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
source_agent_name=data.get("source_agent_name"),
|
||||
execution_agent_name=data.get("execution_agent_name"),
|
||||
name=data["name"],
|
||||
description=data.get("description"),
|
||||
prompt=data["prompt"],
|
||||
schedule_text=data["schedule_text"],
|
||||
cron_expr=data["cron_expr"],
|
||||
|
||||
+94
-4
@@ -18,6 +18,7 @@ from zoneinfo import ZoneInfo
|
||||
from croniter import croniter
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
|
||||
from deerflow.config.agents_config import AGENT_ID_PATTERN
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
@@ -305,12 +306,51 @@ class ScheduledTaskService:
|
||||
"""
|
||||
run = await self.store.get_run(run_id, user_id)
|
||||
if run is None:
|
||||
# Either the run doesn't exist, or the caller is a subscriber, not owner.
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
raise PermissionError("Only the run owner may edit result files")
|
||||
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
|
||||
|
||||
async def admin_update_run_file(
|
||||
self,
|
||||
run_id: str,
|
||||
file_index: int,
|
||||
new_content: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Owner-less variant of :meth:`update_run_file` used by the public viewer.
|
||||
|
||||
The public scheduled-task page is unauthenticated, so it cannot supply a
|
||||
``user_id``. Mutations from that page route here instead. The disk-write
|
||||
and result-row update logic is identical to the owner path.
|
||||
"""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
return await self._apply_run_file_edit(run, run_id, file_index, new_content)
|
||||
|
||||
async def admin_update_run_body(self, run_id: str, new_content: str) -> dict[str, Any]:
|
||||
"""Overwrite the inline Markdown body (``result.content``) of a run.
|
||||
|
||||
Used by the public scheduled-task viewer so anonymous editors can fix
|
||||
typos in the "正文" tab. Unlike :meth:`admin_update_run_file`, no disk
|
||||
artifact backs the body — it lives entirely in the run row.
|
||||
"""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is None:
|
||||
raise FileNotFoundError(f"Run {run_id} not found")
|
||||
result = run.get("result") if isinstance(run.get("result"), dict) else {}
|
||||
new_result = {**result, "content": new_content}
|
||||
await self.store.update_run(run_id, {"result": new_result})
|
||||
return {"content": new_content}
|
||||
|
||||
async def _apply_run_file_edit(
|
||||
self,
|
||||
run: dict[str, Any],
|
||||
run_id: str,
|
||||
file_index: int,
|
||||
new_content: str,
|
||||
) -> dict[str, Any]:
|
||||
result = run.get("result") or {}
|
||||
files = result.get("files") if isinstance(result, dict) else None
|
||||
if not isinstance(files, list) or file_index < 0 or file_index >= len(files):
|
||||
@@ -406,6 +446,7 @@ class ScheduledTaskService:
|
||||
prompt: str,
|
||||
schedule: str,
|
||||
timezone: str | None = None,
|
||||
description: str | None = None,
|
||||
source_thread_id: str | None = None,
|
||||
source_agent_name: str | None = None,
|
||||
execution_agent_name: str | None = None,
|
||||
@@ -420,6 +461,7 @@ class ScheduledTaskService:
|
||||
context = dict(execution_context or {})
|
||||
if execution_agent_name:
|
||||
context["agent_name"] = execution_agent_name
|
||||
normalized_description = description.strip() if isinstance(description, str) else None
|
||||
task = await self.store.create_task(
|
||||
{
|
||||
"task_id": str(uuid.uuid4()),
|
||||
@@ -429,6 +471,7 @@ class ScheduledTaskService:
|
||||
"source_agent_name": source_agent_name,
|
||||
"execution_agent_name": execution_agent_name,
|
||||
"name": final_name,
|
||||
"description": normalized_description or None,
|
||||
"prompt": prompt.strip(),
|
||||
"schedule_text": schedule.strip(),
|
||||
"cron_expr": cron_expr,
|
||||
@@ -451,6 +494,15 @@ class ScheduledTaskService:
|
||||
for key in ("name", "prompt", "enabled", "execution_agent_name"):
|
||||
if key in data and data[key] is not None:
|
||||
update_data[key] = data[key]
|
||||
# Description is nullable — accept ``None`` (clear) and empty strings
|
||||
# (also clear) explicitly, so the field can be removed via PATCH.
|
||||
if "description" in data:
|
||||
raw = data["description"]
|
||||
if isinstance(raw, str):
|
||||
trimmed = raw.strip()
|
||||
update_data["description"] = trimmed or None
|
||||
else:
|
||||
update_data["description"] = None
|
||||
# Names are matched against the per-user uniqueness constraint, so keep
|
||||
# the stored value trimmed to match create_task's normalization.
|
||||
if isinstance(update_data.get("name"), str):
|
||||
@@ -661,7 +713,7 @@ class ScheduledTaskService:
|
||||
retry_count: int = 0,
|
||||
) -> None:
|
||||
"""Mark a run failed and reflect the outcome on the parent task."""
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"定时任务执行失败:{error}")
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
|
||||
@@ -819,8 +871,17 @@ class ScheduledTaskService:
|
||||
try:
|
||||
request = SimpleNamespace(app=self.app)
|
||||
execution_context = dict(task.get("execution_context") or {})
|
||||
if task.get("execution_agent_name"):
|
||||
execution_context["agent_name"] = task["execution_agent_name"]
|
||||
stored_agent_ref = task.get("execution_agent_name")
|
||||
if stored_agent_ref:
|
||||
resolved_agent_id = await self._resolve_agent_id(stored_agent_ref, task["user_id"])
|
||||
if resolved_agent_id:
|
||||
execution_context["agent_name"] = resolved_agent_id
|
||||
else:
|
||||
logger.warning(
|
||||
"Scheduled task %s references unknown agent %r; falling back to default lead agent",
|
||||
task.get("task_id"),
|
||||
stored_agent_ref,
|
||||
)
|
||||
execution_context.update(
|
||||
{
|
||||
"is_scheduled_run": True,
|
||||
@@ -845,6 +906,35 @@ class ScheduledTaskService:
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
async def _resolve_agent_id(self, stored_ref: str, user_id: str) -> str | None:
|
||||
"""Resolve a scheduled task's ``execution_agent_name`` to a stable agent id.
|
||||
|
||||
Older tasks (created before the frontend was fixed) stored the agent's
|
||||
Chinese **display name** here; the runtime validator only accepts ids
|
||||
matching ``^[A-Za-z0-9_-]+$``, so those rows would 422 forever. We
|
||||
gracefully recover by looking the stored value up first as an id, then
|
||||
as a display name. Returns ``None`` when nothing visible matches — the
|
||||
caller then falls back to the default lead agent rather than crashing.
|
||||
"""
|
||||
store = getattr(self.app.state, "agent_store", None)
|
||||
if store is None:
|
||||
return stored_ref if AGENT_ID_PATTERN.fullmatch(stored_ref) else None
|
||||
if AGENT_ID_PATTERN.fullmatch(stored_ref):
|
||||
try:
|
||||
row = await store.get_visible(stored_ref, user_id)
|
||||
except Exception:
|
||||
row = None
|
||||
if row is not None:
|
||||
return row["id"]
|
||||
try:
|
||||
agents = await store.list_visible(user_id)
|
||||
except Exception:
|
||||
return None
|
||||
for agent in agents:
|
||||
if agent.get("name") == stored_ref:
|
||||
return agent.get("id")
|
||||
return None
|
||||
|
||||
async def _extract_latest_ai_text(self, thread_id: str) -> str | None:
|
||||
checkpointer = self.app.state.checkpointer
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
|
||||
+6
-1
@@ -39,6 +39,12 @@ def _extract_json_object(raw: str) -> dict | None:
|
||||
|
||||
async def scan_skill_content(content: str, *, executable: bool = False, location: str = SKILL_MD_FILE, app_config: AppConfig | None = None) -> ScanResult:
|
||||
"""Screen skill content before it is written to disk."""
|
||||
config = app_config or get_app_config()
|
||||
# Operator-level kill switch: when ``skills.security_scan.enabled`` is false,
|
||||
# bypass the LLM round-trip entirely and let the content through.
|
||||
if not config.skills.security_scan.enabled:
|
||||
return ScanResult("allow", "Security scan disabled by config.")
|
||||
|
||||
rubric = (
|
||||
"You are a security reviewer for AI agent skills. "
|
||||
"Classify the content as allow, warn, or block. "
|
||||
@@ -49,7 +55,6 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
|
||||
prompt = f"Location: {location}\nExecutable: {str(executable).lower()}\n\nReview this content:\n-----\n{content}\n-----"
|
||||
|
||||
try:
|
||||
config = app_config or get_app_config()
|
||||
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
|
||||
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
|
||||
model_name = config.skill_evolution.curator.model_name
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from .agent_orchestration_tool import agent_orchestration_tool
|
||||
from .clarification_tool import ask_clarification_tool
|
||||
from .fast_chat_tool import fast_chat_tool, fast_llm_answer
|
||||
from .memory_tool import memory_tool
|
||||
@@ -12,6 +13,7 @@ __all__ = [
|
||||
"setup_agent",
|
||||
"present_file_tool",
|
||||
"ask_clarification_tool",
|
||||
"agent_orchestration_tool",
|
||||
"memory_tool",
|
||||
"scheduled_task_tool",
|
||||
"view_image_tool",
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
"""Placeholder tool that lets the roundtable leader agent dispatch sub-agents.
|
||||
|
||||
Background
|
||||
----------
|
||||
The multi-agent ``/api/multi-agent/run/stream`` endpoint (``app.gateway.routers.multi_agent``)
|
||||
drives a *leader* agent whose only job is to delegate work to seat-specific sub-agents.
|
||||
Its SOUL.md tells the model to "use the agent_orchestration skill, passing agent_name
|
||||
and task". With no real tool registered under that name, the model used to emit
|
||||
``{"name": "agent_orchestration", "args": {"agent_name": ..., "task": ...}}`` calls
|
||||
that the runtime rejected with "agent_orchestration is not a valid tool", driving it
|
||||
into an unproductive retry loop (a single leader call could exceed 10 000 SSE frames).
|
||||
|
||||
This module exposes a no-op tool with the exact signature the leader naturally
|
||||
emits. The actual dispatch is handled by ``SkillStopMiddleware`` which intercepts
|
||||
the call, halts the run, and lets ``multi_agent.py`` parse the AIMessage's
|
||||
``tool_calls`` to extract ``(agent_name, task)`` pairs for the frontend.
|
||||
"""
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
|
||||
@tool("agent_orchestration", parse_docstring=True)
|
||||
def agent_orchestration_tool(agent_name: str, task: str) -> str:
|
||||
"""Delegate a task to a named sub-agent in the multi-agent roundtable.
|
||||
|
||||
Use this tool ONLY when you are the roundtable coordinator and need to
|
||||
dispatch work to one of the configured seat agents. Each call delegates
|
||||
ONE task to ONE agent. Emit multiple calls in the same turn to dispatch
|
||||
several seats in a single round of the discussion.
|
||||
|
||||
The call is intercepted by the runtime — you do not get a real result
|
||||
back synchronously. The orchestrator will collect every dispatched
|
||||
``(agent_name, task)`` pair, run the corresponding sub-agents, then
|
||||
re-prompt you with a synthesis instruction once they finish.
|
||||
|
||||
Args:
|
||||
agent_name: Name of the sub-agent to dispatch. MUST exactly match one of
|
||||
the names listed in your SOUL prompt's ``可调度智能体`` section.
|
||||
task: The instruction to hand to that sub-agent. Be specific — the
|
||||
sub-agent only sees this string (plus its own SOUL prompt), not the
|
||||
broader conversation context.
|
||||
"""
|
||||
# Real handling is in SkillStopMiddleware + the multi_agent router. This
|
||||
# body only runs as a fallback if the middleware is not configured; it
|
||||
# returns a placeholder so the model does not see a "not a valid tool" error.
|
||||
return f"dispatched:{agent_name}"
|
||||
+20
-7
@@ -36,18 +36,31 @@ def ask_clarification_tool(
|
||||
- You're about to perform a potentially dangerous operation
|
||||
- You have a recommendation but need user approval
|
||||
|
||||
Batching multiple independent questions:
|
||||
- **You CAN and SHOULD call ask_clarification multiple times in the same turn** when the user's
|
||||
request leaves several **independent** aspects unclear (e.g. project type AND tech stack AND
|
||||
deployment target). Emit one tool call per aspect — do NOT cram several unrelated questions
|
||||
into one `question` string.
|
||||
- The frontend automatically groups all clarification cards in one turn into a single submission,
|
||||
so the user answers them together in one reply. You will then receive one combined user message
|
||||
addressing every question in order.
|
||||
- When the questions DEPEND on each other (the second only makes sense after the first is answered),
|
||||
ask only the first one and wait for the reply before asking the next.
|
||||
|
||||
Best practices:
|
||||
- Ask ONE clarification at a time for clarity
|
||||
- Be specific and clear in your question
|
||||
- Don't make assumptions when clarification is needed
|
||||
- For risky operations, ALWAYS ask for confirmation
|
||||
- After calling this tool, execution will be interrupted automatically
|
||||
- One question per ask_clarification call — keep each `question` focused on a single decision.
|
||||
- Be specific and clear in your question.
|
||||
- Provide `options` (with concise labels) whenever there is a discrete set of reasonable choices —
|
||||
this lets the user multi-select in the UI instead of typing free text.
|
||||
- Don't make assumptions when clarification is needed.
|
||||
- For risky operations, ALWAYS ask for confirmation.
|
||||
- After calling this tool (or several in one turn), execution will be interrupted automatically.
|
||||
|
||||
Args:
|
||||
question: The clarification question to ask the user. Be specific and clear.
|
||||
question: The clarification question to ask the user. Focused on a single decision; be specific and clear.
|
||||
clarification_type: The type of clarification needed (missing_info, ambiguous_requirement, approach_choice, risk_confirmation, suggestion).
|
||||
context: Optional context explaining why clarification is needed. Helps the user understand the situation.
|
||||
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from.
|
||||
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from; the UI supports multi-select.
|
||||
"""
|
||||
# This is a placeholder implementation
|
||||
# The actual logic is handled by ClarificationMiddleware which intercepts this tool call
|
||||
|
||||
+13
-1
@@ -82,7 +82,19 @@ def memory_tool(
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
if not config.enabled or not config.builtin.enabled:
|
||||
if not config.enabled:
|
||||
return json.dumps(
|
||||
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
return json.dumps(
|
||||
{"success": False, "error": "当前使用 V1 记忆模式,记忆由系统自动提取,不支持手动写入。"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
if not config.builtin.enabled:
|
||||
return json.dumps(
|
||||
{"success": False, "error": "记忆功能未启用,可能在配置中被关闭。"},
|
||||
ensure_ascii=False,
|
||||
|
||||
@@ -7,6 +7,7 @@ from deerflow.config.app_config import AppConfig
|
||||
from deerflow.reflection import resolve_variable
|
||||
from deerflow.sandbox.security import is_host_bash_allowed
|
||||
from deerflow.tools.builtins import ask_clarification_tool, fast_chat_tool, fast_llm_answer, memory_tool, present_file_tool, router_quality_check, scheduled_task_tool, task_tool, view_image_tool
|
||||
from deerflow.tools.builtins import agent_orchestration_tool, ask_clarification_tool, memory_tool, present_file_tool, scheduled_task_tool, task_tool, view_image_tool
|
||||
from deerflow.tools.builtins.tool_search import reset_deferred_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -17,6 +18,7 @@ BUILTIN_TOOLS = [
|
||||
fast_chat_tool,
|
||||
fast_llm_answer,
|
||||
router_quality_check,
|
||||
agent_orchestration_tool,
|
||||
]
|
||||
|
||||
SUBAGENT_TOOLS = [
|
||||
@@ -42,6 +44,7 @@ def get_available_tools(
|
||||
model_name: str | None = None,
|
||||
subagent_enabled: bool = False,
|
||||
is_scheduled_run: bool = False,
|
||||
excluded_tools: list[str] | None = None,
|
||||
*,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> list[BaseTool]:
|
||||
@@ -55,6 +58,9 @@ def get_available_tools(
|
||||
include_mcp: Whether to include tools from MCP servers (default: True).
|
||||
model_name: Optional model name to determine if vision tools should be included.
|
||||
subagent_enabled: Whether to include subagent tools (task, task_status).
|
||||
excluded_tools: Optional list of tool names to exclude from the result
|
||||
(applied AFTER all other resolution). Takes precedence over
|
||||
``app_config.excluded_tools``.
|
||||
|
||||
Returns:
|
||||
List of available tools.
|
||||
@@ -212,4 +218,19 @@ def get_available_tools(
|
||||
"Duplicate tool name %r detected and skipped — check your config.yaml and MCP server registrations (issue #1803).",
|
||||
t.name,
|
||||
)
|
||||
|
||||
# Drop tools the caller (or config) wants excluded. Request-level
|
||||
# ``excluded_tools`` wins; the config-level list is the static fallback.
|
||||
effective_excluded = excluded_tools if excluded_tools else getattr(config, "excluded_tools", None) or []
|
||||
if effective_excluded:
|
||||
excluded_set = set(effective_excluded)
|
||||
filtered_tools = [t for t in unique_tools if t.name not in excluded_set]
|
||||
if len(filtered_tools) < len(unique_tools):
|
||||
logger.info(
|
||||
"Excluded %d tool(s) by request/config: %s",
|
||||
len(unique_tools) - len(filtered_tools),
|
||||
effective_excluded,
|
||||
)
|
||||
unique_tools = filtered_tools
|
||||
|
||||
return unique_tools
|
||||
Reference in new issue
Block a user