Merge branch 'ji' into dhk-1

Resolve column_additions_mysql.sql conflict: keep square_id columns from master and roundtable_drafts.task_id from dhk-1.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
董宏凯andCursor committed 2026-06-11 17:40:08 +08:00
commit bcc3d617a0
142 files changed
+14953 -2383

No files matched your search

@@ -109,20 +109,42 @@ async def _search_notebook(
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
import httpx # 懒导入:仅在笔记模式下才需要
# 入参全 dump(token 只打长度,不打明文,防止泄漏到日志)
logger.info(
"[notebook-search] called keywords=%s search_space_id=%s source_ids=%s token_len=%d",
keywords,
search_space_id,
source_ids,
len(token) if token else 0,
)
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
if not base:
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
# 改成 error 级别,warning 在很多部署里默认会被过滤掉看不到
logger.error(
"[notebook-search] DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search. "
"请在后端启动环境里设这个环境变量(指向笔记后端根地址,例如 http://localhost:8003)。"
)
return []
full_url = f"{base}{_NOTEBOOK_SEARCH_PATH}"
logger.info("[notebook-search] base=%s full_url=%s", base, full_url)
results: list[Material] = []
mat_idx = 0
try:
async with httpx.AsyncClient(timeout=20) as client:
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
"query": kw,
"source_ids": source_ids,
"search_space_id": search_space_id,
"top_k": _NOTEBOOK_TOP_K,
})
try:
# /mentioned-documents/search 接口无需鉴权,不传 Authorization header
resp = await client.post(
f"{base}{_NOTEBOOK_SEARCH_PATH}",
headers={"Authorization": f"Bearer {token}"},
full_url,
json={
"query": kw,
"source_ids": source_ids,
@@ -130,10 +152,29 @@ async def _search_notebook(
"top_k": _NOTEBOOK_TOP_K,
},
)
logger.info(
"[notebook-search] kw=%r http_status=%d body_len=%d",
kw,
resp.status_code,
len(resp.content or b""),
)
if resp.status_code >= 400:
# 把错误响应体打出来,方便定位 401 / 422 等
logger.error(
"[notebook-search] kw=%r HTTP %d response: %s",
kw,
resp.status_code,
(resp.text or "")[:1000],
)
resp.raise_for_status()
data = resp.json()
# 解析 documents 列表,每个 chunk 作为独立素材
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
logger.info(
"[notebook-search] kw=%r parsed documents=%d",
kw,
len(documents or []),
)
for doc in documents or []:
doc_title: str = doc.get("title") or "笔记文档"
doc_url: str = doc.get("url") or ""
@@ -154,10 +195,11 @@ async def _search_notebook(
url=doc_url,
))
except Exception as e:
logger.warning("notebook search kw=%r failed: %s", kw, e)
logger.warning("[notebook-search] kw=%r failed: %s", kw, e, exc_info=True)
except Exception as e:
logger.warning("notebook search client error: %s", e)
logger.warning("[notebook-search] client error: %s", e, exc_info=True)
logger.info("[notebook-search] done — returning %d materials", len(results))
return results
@@ -250,6 +292,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
notebook_materials: list[Material] = []
if notebook_search_space_id:
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
notebook_token: str = cfg.get("notebook_token") or ""
@@ -369,6 +412,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
all_results_nested: list = []
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
elif material_source == "notebook" and notebook_search_space_id:
# ── 纯笔记模式 ──
# 用户在表单选了「我的空间」作为素材来源 → 跳过 web/intranet 检索,
# 仅使用 Step 1.5 已拉取的 notebook_materials。
# 与 UI 文案"从空间中检索素材"对齐。
_emit(f"仅使用笔记素材,跳过 web/intranet 检索({len(notebook_materials)} 条)")
all_results_nested = []
else:
# ── 常规 web/intranet 模式 ──
provider = get_search_provider(ai_writing_config)
@@ -20,7 +20,7 @@ from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddlew
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
from deerflow.agents.thread_state import ThreadState
from deerflow.config.agents_config import load_agent_config, validate_agent_id
from deerflow.config.agents_config import load_agent_config, load_agent_soul, validate_agent_id
from deerflow.config.app_config import AppConfig, get_app_config
from deerflow.models import create_chat_model
@@ -331,6 +331,13 @@ def _build_middlewares(
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents))
# SkillReviewMiddleware — trigger background skill review after each turn.
# Gated by skill_evolution.enabled + skill_evolution.review_worker.enabled.
if resolved_app_config.skill_evolution.enabled and resolved_app_config.skill_evolution.review_worker.enabled:
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
middlewares.append(SkillReviewMiddleware())
# LoopDetectionMiddleware — detect and break repetitive tool call loops
middlewares.append(LoopDetectionMiddleware())
@@ -535,11 +542,30 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
default_tools = [notebook_search_tool] + default_tools
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
tools=default_tools,
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
system_prompt=apply_prompt_template(
# Custom agents are steered ONLY by their own SOUL.md — no generic
# super-agent system template and no personal memory injection. We branch to
# the agent-only prompt builder whenever a custom agent config carries a
# non-empty SOUL.md. The plain default lead agent (no agent_config) and
# notebook mode keep the full template. See apply_agent_only_prompt_template.
has_agent_soul = bool(agent_config) and bool(load_agent_soul(agent_id))
use_agent_only_prompt = has_agent_soul and not is_notebook_mode
if use_agent_only_prompt:
from deerflow.agents.lead_agent.prompt import apply_agent_only_prompt_template
system_prompt = apply_agent_only_prompt_template(
agent_id=agent_id,
agent_name=agent_display_name,
subagent_enabled=subagent_enabled,
max_concurrent_subagents=max_concurrent_subagents,
available_skills=(set(agent_config.skills or []) if agent_config else None),
app_config=resolved_app_config,
rag_mode_enabled=rag_mode_enabled,
writing_mode=writing_mode,
writing_artifact_path=writing_artifact_path,
)
else:
system_prompt = apply_prompt_template(
subagent_enabled=subagent_enabled,
max_concurrent_subagents=max_concurrent_subagents,
agent_id=agent_id,
@@ -552,6 +578,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
rag_mode_enabled=rag_mode_enabled,
is_notebook_mode=is_notebook_mode,
notebook_source_ids=notebook_source_ids,
),
)
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
tools=default_tools,
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
system_prompt=system_prompt,
state_schema=ThreadState,
)
@@ -615,6 +615,127 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
"""
# Minimal system prompt for custom agents that should be steered ONLY by their
# own SOUL.md — no generic "super agent" persona scaffolding (role line, thinking
# style, clarification system, citation rules, response-style / critical-reminder
# defaults) and no personal memory injection. Only the mechanical sections an
# agent still needs to actually use its configured skills/tools are kept.
AGENT_ONLY_PROMPT_TEMPLATE = """{soul}
{network_policy_section}{skills_section}
{deferred_tools_section}
{writing_mode_section}
{subagent_section}
<working_directory existed="true">
- User uploads: `/mnt/user-data/uploads` - Files uploaded by the user (automatically listed in context)
- User workspace: `/mnt/user-data/workspace` - Working directory for temporary files
- Output files: `/mnt/user-data/outputs` - Final deliverables must be saved here
**File Management:**
- Uploaded files are automatically listed in the <uploaded_files> section before each request
- Use `read_file` tool to read uploaded files using their paths from the list
- For PDF, PPT, Excel, and Word files, converted Markdown versions (*.md) are available alongside originals
- Treat `/mnt/user-data/workspace` as your default current working directory for coding and file-editing tasks
- Final deliverables must be copied to `/mnt/user-data/outputs` and presented using `present_files` tool
{acp_section}
</working_directory>
"""
def apply_agent_only_prompt_template(
*,
agent_id: str | None,
agent_name: str | None,
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
available_skills: set[str] | None = None,
app_config: AppConfig | None = None,
rag_mode_enabled: bool = False,
writing_mode: bool = False,
writing_artifact_path: str | None = None,
) -> str:
"""Build a custom agent's system prompt from ONLY its SOUL.md plus the
mechanical sections required for its configured skills/tools to work.
Unlike :func:`apply_prompt_template`, this deliberately omits:
- the generic ``<role>`` "internal super agent" framing,
- ``<thinking_style>`` / ``<clarification_system>`` / ``<response_style>`` /
``<citations>`` / ``<critical_reminders>`` behavioural defaults,
- the personal ``<memory>`` injection (USER.md / MEMORY.md).
The agent's SOUL.md is therefore the sole source of persona and behaviour.
Skills, deferred tools, the subagent orchestration section (when enabled) and
the working-directory contract are kept because they are capability wiring,
not persona — dropping them would silently break agents that rely on skills
or file output.
"""
storage_agent_id = agent_id if agent_id is not None else agent_name
soul = get_agent_soul(storage_agent_id)
skills_section = get_skills_prompt_section(
available_skills,
app_config=app_config,
display_enabled=rag_mode_enabled,
)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
n = max_concurrent_subagents
subagent_section = _build_subagent_section(n, app_config=app_config) if subagent_enabled else ""
current_writing_artifact = (
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
if writing_artifact_path
else ""
)
writing_mode_section = (
f"""<writing_mode>
Writing mode is enabled for this run.
- Your final deliverable MUST be a Markdown file saved under `/mnt/user-data/outputs/` (e.g. `/mnt/user-data/outputs/report.md`).
- If a current Markdown artifact is provided below, revise that file rather than creating a new one unless the user explicitly asks for a separate document.
- After writing the file, call `present_files` with the Markdown file path.
{current_writing_artifact}</writing_mode>
"""
if writing_mode
else ""
)
acp_section = _build_acp_section(app_config=app_config)
custom_mounts_section = _build_custom_mounts_section(app_config=app_config)
acp_and_mounts_section = "\n".join(section for section in (acp_section, custom_mounts_section) if section)
network_policy_section = _build_network_policy_section()
rag_mode_section = (
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering per search term, tool, or skill.
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
- Do not output `<skill-display .../>` in RAG/reference mode; the frontend renders the right reference panel.
</rag_citation_mode>
"""
if rag_mode_enabled
else ""
)
prompt = AGENT_ONLY_PROMPT_TEMPLATE.format(
soul=soul,
skills_section=skills_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
subagent_section=subagent_section,
network_policy_section=network_policy_section,
acp_section=acp_and_mounts_section,
)
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
return prompt
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
"""组装注入系统提示词的记忆上下文 (Memory V2)。
@@ -637,28 +758,23 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
from deerflow.config.memory_config import get_effective_memory_config
from deerflow.runtime.user_context import get_effective_user_id
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
if not config.enabled or not config.injection.enabled:
return ""
manager = build_memory_manager(
user_id=user_id,
agent_id=agent_name or "default",
config=config,
)
user_id = get_effective_user_id()
config = get_effective_memory_config(user_id)
manager = build_memory_manager(
user_id=user_id,
agent_id=agent_name or "default",
config=config,
app_config=app_config,
)
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
memory_content = manager.build_system_prompt()
if not memory_content.strip():
return ""
return f"""<memory>
{memory_content}
</memory>
"""
context_content = manager.build_system_prompt()
if not context_content.strip():
return ""
return f"{context_content}\n"
except Exception:
logger.exception("Failed to load memory context")
return ""
@@ -1078,7 +1194,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
<notebook_mode>
## 角色
你是一个笔记本 AI 助手,**只能**基于笔记本文档来源回答用户问题,禁止使用任何其他知识。
你是一个空间 AI 助手,**只能**基于空间文档来源回答用户问题,禁止使用任何其他知识。
## 可用知识来源
{source_line}
@@ -1104,7 +1220,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
- 未调用 `search_notebook_sources` 就直接回答
- 引用不在检索结果中的 chunk_id
- 省略引用标记
- 使用笔记本来源之外的知识
- 使用空间来源之外的知识
</notebook_mode>"""
return prompt
@@ -32,6 +32,8 @@ logger = logging.getLogger(__name__)
#: 内建 provider 的固定名字。
BUILTIN_PROVIDER_NAME = "builtin"
KNOWLEDGE_PROVIDER_NAME = "knowledge"
_INTERNAL_PROVIDER_NAMES = {BUILTIN_PROVIDER_NAME, KNOWLEDGE_PROVIDER_NAME}
class MemoryManager:
@@ -49,11 +51,11 @@ class MemoryManager:
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
第二个会被拒绝并 log 警告。返回是否注册成功。
"""
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
if not is_builtin:
is_internal = provider.name in _INTERNAL_PROVIDER_NAMES
if not is_internal:
if self._has_external:
existing = next(
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
(p.name for p in self._providers if p.name not in _INTERNAL_PROVIDER_NAMES),
"unknown",
)
logger.warning(
@@ -220,6 +222,7 @@ def build_memory_manager(
agent_id: str | None = None,
thread_id: str | None = None,
config: Any = None,
app_config: Any = None,
) -> MemoryManager:
"""按生效配置组装并初始化一个 MemoryManager。
@@ -246,7 +249,9 @@ def build_memory_manager(
manager = MemoryManager()
base_dir = str(get_paths().base_dir)
if getattr(config, "version", "v2") == "v1":
memory_enabled = bool(getattr(config, "enabled", True) and getattr(config.injection, "enabled", True))
if memory_enabled and getattr(config, "version", "v2") == "v1":
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
@@ -260,7 +265,7 @@ def build_memory_manager(
token_budget=v1_cfg.token_budget,
)
)
else:
elif memory_enabled:
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
@@ -286,11 +291,21 @@ def build_memory_manager(
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
try:
from deerflow.agents.memory.providers.knowledge import KnowledgeProvider
knowledge = KnowledgeProvider(app_config=app_config)
if knowledge.is_available():
manager.add_provider(knowledge)
except Exception as exc: # noqa: BLE001 - 知识库路由失败不能阻断对话
logger.warning("加载 Knowledge provider 失败:%s", exc)
manager.initialize_all(
user_id=user_id,
agent_id=agent_id,
thread_id=thread_id,
base_dir=base_dir,
app_config=app_config,
)
return manager
@@ -227,7 +227,10 @@ class BuiltinFileProvider(MemoryProvider):
保持系统提示词稳定,利于 prefix cache。
"""
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
return "\n\n".join(blocks)
content = "\n\n".join(blocks)
if not content:
return ""
return f"<memory>\n{content}\n</memory>"
def get_tool_schemas(self) -> list[dict[str, Any]]:
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
@@ -0,0 +1,93 @@
"""Read-only provider for the mounted policy knowledge base.
This provider keeps the current knowledge-base behaviour: inject the lightweight
``routing.md`` guide and let the model decide when to read concrete files. It
does not automatically retrieve snippets and never writes to the knowledge base.
"""
from __future__ import annotations
import logging
from functools import lru_cache
from pathlib import Path
from typing import Any
from deerflow.agents.memory.provider import MemoryProvider
logger = logging.getLogger(__name__)
@lru_cache(maxsize=16)
def _read_routing_file(host_path: str) -> str:
path = Path(host_path)
if not path.is_file():
return ""
return path.read_text(encoding="utf-8").strip()
class KnowledgeProvider(MemoryProvider):
"""Injects the knowledge routing guide as a read-only context provider."""
def __init__(self, *, app_config: Any | None = None) -> None:
self._app_config = app_config
self._routing_host_path: Path | None = None
@property
def name(self) -> str:
return "knowledge"
def is_available(self) -> bool:
return True
def initialize(
self,
*,
user_id: str,
agent_id: str | None = None,
thread_id: str | None = None,
base_dir: str,
**kwargs: Any,
) -> None:
app_config = self._app_config or kwargs.get("app_config")
if app_config is None:
from deerflow.config import get_app_config
app_config = get_app_config()
self._routing_host_path = None
for mount in app_config.sandbox.mounts or []:
if mount.container_path.rstrip("/") == "/mnt/knowledge":
self._routing_host_path = Path(mount.host_path) / "routing.md"
break
def system_prompt_block(self) -> str:
if self._routing_host_path is None:
return ""
content = _read_routing_file(str(self._routing_host_path.resolve()))
if not content:
return ""
logger.debug("Injected knowledge routing memory from %s", self._routing_host_path)
return f"""<knowledge_routing_memory source="/mnt/knowledge/routing.md">
{content}
</knowledge_routing_memory>"""
def prefetch(self, query: str, *, thread_id: str = "") -> str:
return ""
def sync_turn(
self,
user_content: str,
assistant_content: str,
*,
thread_id: str = "",
turn_index: int | None = None,
) -> None:
return None
def get_tool_schemas(self) -> list[dict[str, Any]]:
return []
__all__ = ["KnowledgeProvider"]
@@ -0,0 +1,54 @@
"""SkillReviewMiddleware — triggers background skill review after each agent turn.
Hooks into ``after_agent``: once the lead agent graph has finished a full turn,
this middleware counts the tool calls made and, if the threshold is met, hands
the conversation messages to the SkillReviewWorker as a background asyncio task.
No blocking I/O or LLM calls happen in-band — the review runs fully in the
background and never delays the HTTP response.
"""
from __future__ import annotations
import logging
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langgraph.runtime import Runtime
from deerflow.config.app_config import get_app_config
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.skills.review_worker import maybe_start_review
logger = logging.getLogger(__name__)
class SkillReviewMiddlewareState(AgentState):
"""Minimal state schema — only needs ``messages``."""
pass
class SkillReviewMiddleware(AgentMiddleware[SkillReviewMiddlewareState]):
"""Dispatch a background skill review after every substantial agent turn.
'Substantial' is defined by ``skill_evolution.review_worker.min_tool_calls``
(default 5). Turns below the threshold are silently skipped to avoid
wasting LLM tokens on trivial exchanges.
"""
state_schema = SkillReviewMiddlewareState
@override
def after_agent(self, state: Any, runtime: Runtime) -> None:
"""Fire background review; never raises — failures are logged only."""
try:
app_config = get_app_config()
messages = state.get("messages", []) if hasattr(state, "get") else []
thread_id: str = runtime.context.get("thread_id", "") if runtime.context else ""
if not thread_id:
return
user_id = get_effective_user_id()
maybe_start_review(thread_id, messages, app_config, user_id)
except Exception:
logger.exception("SkillReviewMiddleware.after_agent: unexpected error (ignored)")
@@ -1,21 +1,41 @@
"""Summarization middleware extensions for DeerFlow."""
"""Summarization middleware extensions for DeerFlow.
Unlike LangChain's stock ``SummarizationMiddleware`` (which runs in
``before_model`` and **persists** a destructive ``RemoveMessage(REMOVE_ALL_MESSAGES)``
to the checkpoint — wiping the original Q&A from both the model context *and*
the conversation the frontend renders), this middleware compresses the history
**transiently in ``wrap_model_call``**. The full original messages stay in state
(so the UI keeps showing the user's real conversation); only the message list
handed to the model for a single call is replaced with ``[summary, *recent]``.
The generated summary is cached per-thread so we don't pay an extra LLM call on
every model invocation: once a thread is summarized, the cached summary is reused
until enough *new* messages accumulate to push the compressed context back over
the trigger threshold (sticky boundary, mirroring the stock middleware's
post-compaction behavior).
"""
from __future__ import annotations
import logging
from collections.abc import Collection
from collections import OrderedDict
from collections.abc import Awaitable, Callable, Collection
from dataclasses import dataclass
from typing import Any, Protocol, override, runtime_checkable
from langchain.agents import AgentState
from langchain.agents.middleware import SummarizationMiddleware
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, RemoveMessage, ToolMessage
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, ToolMessage
from langgraph.config import get_config
from langgraph.graph.message import REMOVE_ALL_MESSAGES
from langgraph.runtime import Runtime
logger = logging.getLogger(__name__)
# Upper bound on the per-thread summary cache so a long-lived process serving
# many threads can't grow it without limit. Oldest entries are evicted first.
_SUMMARY_CACHE_MAX_THREADS = 512
@dataclass(frozen=True)
class SummarizationEvent:
@@ -95,6 +115,22 @@ class _SkillBundle:
skill_key: str
@dataclass(frozen=True)
class _SummaryCacheEntry:
"""A cached, reusable summary for one thread.
``summarized_ids`` are the ids of the original messages folded into
``summary_text``. The cache is only reusable while every one of those ids is
still present in the live message list and no AIMessage/ToolMessage pair was
split when the boundary was drawn (``reusable``) — otherwise filtering by id
could orphan a tool result, so we fall back to re-summarizing.
"""
summarized_ids: frozenset[str]
summary_text: str
reusable: bool
class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
"""Summarization middleware with pre-compression hook dispatch and skill rescue."""
@@ -116,69 +152,181 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
self._preserve_recent_skill_count = max(0, preserve_recent_skill_count)
self._preserve_recent_skill_tokens = max(0, preserve_recent_skill_tokens)
self._preserve_recent_skill_tokens_per_skill = max(0, preserve_recent_skill_tokens_per_skill)
# Per-thread cache of the most recent summary: thread_id -> (summarized message ids, summary text).
# Lets us reuse a summary across model calls instead of re-running the LLM every invocation.
self._summary_cache: OrderedDict[str, _SummaryCacheEntry] = OrderedDict()
@override
def before_model(self, state: AgentState, runtime: Runtime) -> dict | None:
return self._maybe_summarize(state, runtime)
"""Disable the stock middleware's destructive ``before_model`` compaction.
Compression now happens transiently in :meth:`wrap_model_call`, so the
persisted conversation (and the frontend's view of it) is never mutated.
"""
return None
@override
async def abefore_model(self, state: AgentState, runtime: Runtime) -> dict | None:
return await self._amaybe_summarize(state, runtime)
"""Async counterpart of :meth:`before_model` — also a no-op."""
return None
def _maybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
messages = state["messages"]
self._ensure_message_ids(messages)
# ---- wrap_model_call: transient, display-preserving compression ----------
total_tokens = self.token_counter(messages)
if not self._should_summarize(messages, total_tokens):
@override
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
compressed = self._compressed_messages(request)
if compressed is not None:
request = request.override(messages=compressed)
return handler(request)
@override
async def awrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
compressed = await self._acompressed_messages(request)
if compressed is not None:
request = request.override(messages=compressed)
return await handler(request)
def _compressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
"""Build the compressed message list for this call (sync summary path).
Returns ``None`` when no compression is needed (the request is passed
through untouched).
"""
plan = self._plan_compression(request)
if plan is None:
return None
cutoff_index = self._determine_cutoff_index(messages)
if cutoff_index <= 0:
return None
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
action, payload = plan
if action == "reuse":
return payload # type: ignore[return-value]
# action == "summarize"
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
summary = self._create_summary(messages_to_summarize)
new_messages = self._build_new_messages(summary)
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
return {
"messages": [
RemoveMessage(id=REMOVE_ALL_MESSAGES),
*new_messages,
*preserved_messages,
]
}
async def _acompressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
"""Async counterpart of :meth:`_compressed_messages`."""
plan = self._plan_compression(request)
if plan is None:
return None
action, payload = plan
if action == "reuse":
return payload # type: ignore[return-value]
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
summary = await self._acreate_summary(messages_to_summarize)
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
async def _amaybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
messages = state["messages"]
def _plan_compression(
self,
request: ModelRequest,
) -> tuple[str, Any] | None:
"""Decide what to do for this model call, doing **no** LLM work.
Returns one of:
- ``None`` — pass the request through unchanged (no compression needed).
- ``("reuse", compressed_messages)`` — reuse the cached summary.
- ``("summarize", (messages_to_summarize, preserved, thread_id))`` — the
caller must run the (a)sync summarizer, then call
:meth:`_finish_summarize`.
"""
messages = list(getattr(request, "messages", None) or [])
if not messages:
return None
self._ensure_message_ids(messages)
thread_id = self._resolve_request_thread_id(request)
total_tokens = self.token_counter(messages)
if not self._should_summarize(messages, total_tokens):
cached = self._summary_cache.get(thread_id) if thread_id else None
present_ids = {m.id for m in messages}
if cached is not None and cached.reusable and cached.summarized_ids <= present_ids:
preserved = [m for m in messages if m.id not in cached.summarized_ids]
effective = [self._summary_message(cached.summary_text), *preserved]
if not self._should_summarize(effective, self.token_counter(effective)):
# Refresh LRU recency and reuse without an LLM call.
self._summary_cache.move_to_end(thread_id)
return ("reuse", effective)
# New messages have pushed the compressed context back over the
# threshold — fall through and re-summarize a larger prefix.
if not self._should_summarize(messages, self.token_counter(messages)):
return None
cutoff_index = self._determine_cutoff_index(messages)
if cutoff_index <= 0:
return None
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
summary = await self._acreate_summary(messages_to_summarize)
new_messages = self._build_new_messages(summary)
messages_to_summarize, preserved = self._partition_with_skill_rescue(messages, cutoff_index)
if not messages_to_summarize:
return None
return {
"messages": [
RemoveMessage(id=REMOVE_ALL_MESSAGES),
*new_messages,
*preserved_messages,
]
}
return ("summarize", (messages_to_summarize, preserved, thread_id))
def _finish_summarize(
self,
messages_to_summarize: list[AnyMessage],
preserved: list[AnyMessage],
summary: str,
thread_id: str | None,
) -> list[AnyMessage]:
"""Assemble the compressed message list and update the per-thread cache."""
compressed = [self._summary_message(summary), *preserved]
if thread_id:
summarized_ids = frozenset(m.id for m in messages_to_summarize if m.id is not None)
preserved_ids = {m.id for m in preserved if m.id is not None}
# A non-empty overlap means skill rescue split an AIMessage across the
# boundary; reconstructing ``preserved`` by id filtering later could
# orphan a tool result, so mark the entry non-reusable (re-summarize).
reusable = summarized_ids.isdisjoint(preserved_ids)
self._store_cache_entry(
thread_id,
_SummaryCacheEntry(summarized_ids=summarized_ids, summary_text=summary, reusable=reusable),
)
return compressed
def _store_cache_entry(self, thread_id: str, entry: _SummaryCacheEntry) -> None:
"""Insert/refresh a cache entry, evicting the oldest thread if over capacity."""
self._summary_cache[thread_id] = entry
self._summary_cache.move_to_end(thread_id)
while len(self._summary_cache) > _SUMMARY_CACHE_MAX_THREADS:
self._summary_cache.popitem(last=False)
@staticmethod
def _resolve_request_thread_id(request: ModelRequest) -> str | None:
"""Best-effort thread id from the model request's runtime."""
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
return _resolve_thread_id(runtime)
def _summary_message(self, summary: str) -> HumanMessage:
"""Build the summary message handed to the model.
It is named ``summary`` and flagged ``hide_from_ui`` defensively. Because
it only ever lives in the transient model request (never in persisted
state), the frontend never receives it — the user keeps seeing the
original conversation.
"""
return HumanMessage(
content=f"Here is a summary of the conversation to date:\n\n{summary}",
name="summary",
additional_kwargs={"hide_from_ui": True, "lc_source": "summarization"},
)
@override
def _build_new_messages(self, summary: str) -> list[HumanMessage]:
"""Override the base implementation to let the human message with the special name 'summary'.
And this message will be ignored to display in the frontend, but still can be used as context for the model.
"""
return [HumanMessage(content=f"Here is a summary of the conversation to date:\n\n{summary}", name="summary")]
"""Kept for backwards compatibility; delegates to :meth:`_summary_message`."""
return [self._summary_message(summary)]
def _partition_with_skill_rescue(
self,
@@ -107,7 +107,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
content=last_message.content,
id=last_message.id,
name=last_message.name,
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
additional_kwargs={**last_message.additional_kwargs, "run_id": (runtime.context or {}).get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
)
return {
@@ -7,7 +7,7 @@ import threading
from typing import Any
import yaml
from pydantic import BaseModel
from pydantic import BaseModel, model_validator
from deerflow.config.paths import get_paths
@@ -56,6 +56,32 @@ def validate_agent_name(name: str | None) -> str | None:
return validate_agent_id(name)
class CommonQuestion(BaseModel):
"""A recommended question shown on the agent chat home / sidebar.
``title`` is the short label rendered on the question chip; ``prompt`` is the
full message actually sent when the user clicks it. Splitting the two lets an
admin keep the chip terse while still sending a richly-worded question.
Legacy configs stored questions as bare strings (content only). The
before-validator coerces such a string into ``{title, prompt}`` so old
config.yaml files keep loading without a migration — the title falls back to
a truncated copy of the content.
"""
title: str = ""
prompt: str = ""
@model_validator(mode="before")
@classmethod
def _coerce_legacy_string(cls, value: Any) -> Any:
if isinstance(value, str):
text = value.strip()
title = text if len(text) <= 40 else text[:40].rstrip() + "…"
return {"title": title, "prompt": text}
return value
class AgentConfig(BaseModel):
"""Configuration for a custom agent."""
@@ -69,6 +95,11 @@ class AgentConfig(BaseModel):
# - [] (explicit empty list): no skills
# - ["skill1", "skill2"]: load only the specified skills
skills: list[str] | None = None
# Frequently-asked questions ("常用问题") shown in the agent chat sidebar.
# Display/UX metadata only — the runtime never reads these. None = none set.
# Accepts both the legacy ``list[str]`` shape and the new ``{title, prompt}``
# objects (see CommonQuestion).
common_questions: list[CommonQuestion] | None = None
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
@@ -1,30 +1,39 @@
from pydantic import BaseModel, Field
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
enabled: bool = Field(
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
from pydantic import BaseModel, Field
class ReviewWorkerConfig(BaseModel):
"""Configuration for the post-turn background skill review worker."""
enabled: bool = Field(default=False, description="Whether the review worker is active.")
min_tool_calls: int = Field(default=5, description="Minimum tool calls in a turn to trigger review. Matches the prompt's '≥5 steps' hard condition so simple queries never reach the LLM reviewer.")
max_messages: int = Field(default=60, description="Max messages passed to the review LLM.")
model_name: str | None = Field(default=None, description="Model to use for review; defaults to primary model.")
timeout_seconds: int = Field(default=120, description="Overall timeout for a single review run.")
class CuratorConfig(BaseModel):
"""Configuration for the background skill curator."""
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
class SkillEvolutionConfig(BaseModel):
"""Configuration for agent-managed skill evolution."""
enabled: bool = Field(
default=False,
description="Whether the agent can create and modify skills under skills/custom.",
)
curator: CuratorConfig = Field(
default_factory=CuratorConfig,
description="Configuration for the background skill curator.",
)
review_worker: ReviewWorkerConfig = Field(
default_factory=ReviewWorkerConfig,
description="Configuration for the post-turn background skill review worker.",
)
@@ -135,8 +135,18 @@ def resolve_user_path(user_id: str) -> Path:
return get_paths().user_dir(user_id) / _FILENAME
# Only these curator sub-fields may be overridden per user.
# All other fields (enabled, model_name, timeouts, etc.) are global-only.
_USER_ALLOWED_CURATOR_KEYS = {"interval_hours", "stale_after_days", "archive_after_days", "model_name"}
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid).
Only the three curator threshold keys are honoured; any other keys
(e.g. legacy ``enabled`` or ``model_name``) are silently stripped so that
old files written by previous versions cannot override global-only settings.
"""
path = resolve_user_path(user_id)
if not path.exists():
return None
@@ -154,7 +164,14 @@ def load_user_overrides(user_id: str) -> dict[str, Any] | None:
if not isinstance(data, dict):
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
return None
return data
# Strip top-level keys that are not user-configurable (e.g. legacy "enabled").
curator_raw = data.get("curator")
if not isinstance(curator_raw, dict):
return None
filtered_curator = {k: v for k, v in curator_raw.items() if k in _USER_ALLOWED_CURATOR_KEYS}
if not filtered_curator:
return None
return {"curator": filtered_curator}
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
@@ -86,6 +86,75 @@ class PrivateSquareSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
#: Id of the always-present built-in default square (used when the admin has
#: configured no squares, or when a published item picks none).
DEFAULT_SQUARE_ID = "default"
class PublishSquare(BaseModel):
"""One admin-configured publish square (发布广场).
A pure organizational category that published agents/skills can be filed
under — unlike the password-gated :class:`PrivateSquareSettings`, a square
has **no access control**: every logged-in user can browse every square.
Each agent/skill belongs to at most one square (a single ``square_id``
field on the row), so an item can never be "published twice".
"""
id: str = Field(..., description="Stable square id (slug/uuid); used as agents.square_id / skills.square_id")
name: str = Field(..., description="Display name shown as the square's tab label")
description: str = Field(default="", description="Optional description shown in the admin config / tab hint")
model_config = ConfigDict(extra="ignore")
class PublishSquaresSettings(BaseModel):
"""Admin-configurable list of publish squares.
``squares`` is display-ordered. ``default_square_id`` selects the square a
publisher lands in when they do not pick one explicitly. There is always at
least one effective square — :func:`list_publish_squares` injects a built-in
"默认广场" when the admin has configured none, so the publish flow always has
a valid target.
"""
squares: list[PublishSquare] = Field(default_factory=list, description="Ordered list of configured squares")
default_square_id: str = Field(default=DEFAULT_SQUARE_ID, description="Square id used when a publisher picks none")
model_config = ConfigDict(extra="ignore")
def list_publish_squares(settings: PublishSquaresSettings) -> list[PublishSquare]:
"""Return the effective square list, always non-empty.
Falls back to a single built-in default square when the admin has not
configured any, so callers never have to special-case the empty state.
"""
if settings.squares:
return list(settings.squares)
return [PublishSquare(id=DEFAULT_SQUARE_ID, name="默认广场", description="")]
def resolve_default_square_id(settings: PublishSquaresSettings) -> str:
"""Resolve the effective default square id against the configured set."""
ids = [s.id for s in settings.squares]
if settings.default_square_id and settings.default_square_id in ids:
return settings.default_square_id
return ids[0] if ids else DEFAULT_SQUARE_ID
def normalize_square_id(square_id: str | None, settings: PublishSquaresSettings) -> str:
"""Coerce an arbitrary ``square_id`` to a valid configured square id.
Empty / unknown / orphaned (square later deleted) ids all collapse to the
effective default square, so a published item always resolves to a real
tab.
"""
default_id = resolve_default_square_id(settings)
if not square_id:
return default_id
valid = {s.id for s in list_publish_squares(settings)}
return square_id if square_id in valid else default_id
class LeaderboardSettings(BaseModel):
"""Admin-configurable defaults for the user leaderboard analytics."""
@@ -102,6 +171,7 @@ class SystemSettings(BaseModel):
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
publish_squares: PublishSquaresSettings = Field(default_factory=PublishSquaresSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
@@ -135,7 +135,15 @@ def to_cypher(graph: dict[str, Any]) -> str:
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
can stay in sync with the app's day/night mode (an iframe cannot inherit the
parent's CSS). The layout settles and then **freezes** so a dense graph stops
drifting and stays clickable; node size scales with degree; and hovering or
clicking a node focuses it + its direct neighbours while dimming everything
else, which cuts through heavy node/edge clutter.
"""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
@@ -178,50 +186,250 @@ _HTML_TEMPLATE = """<!DOCTYPE html>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
canvas{display:block}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
:root{
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
}
html.light{
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
}
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
canvas{display:block;cursor:grab}
canvas.grabbing{cursor:grabbing}
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
#legend .dot{width:10px;height:10px;border-radius:50%}
#tip{padding:4px 8px;pointer-events:none;display:none}
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
#panel .meta{color:var(--sub);margin-bottom:8px}
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
#reset:hover{filter:brightness(1.1)}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
</style>
</head>
<body>
<div id="hud"></div>
<div id="tip"></div>
<div id="hud" class="card"></div>
<div id="legend" class="card">
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
</div>
<div id="tip" class="card"></div>
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
<div id="reset" class="card">重置视图</div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
const PAL = {
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
}[THEME];
document.documentElement.classList.toggle('light', THEME === 'light');
document.getElementById('lg-note').style.background = PAL.note;
document.getElementById('lg-entity').style.background = PAL.entity;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
let W=0,H=0;
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
addEventListener('resize',resize);resize();
const panel = document.getElementById('panel');
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
addEventListener('resize',()=>{resize();bump();});resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
let cam={x:0,y:0,k:1},drag=null,pan=null;
function step(){
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
// --- index, adjacency, degree, node sizing -------------------------------
const idx={}, adj={};
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
const edges=[];
for(const e of GRAPH.edges){
const s=idx[e.source], t=idx[e.target];
if(!s||!t||s===t) continue;
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
adj[e.source].add(e.target); adj[e.target].add(e.source);
}
function draw(){
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
ctx.strokeStyle='rgba(120,150,255,.25)';
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
const N=GRAPH.nodes.length;
GRAPH.nodes.forEach((n,i)=>{
n.deg=adj[n.id].size;
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
});
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
const REP=9000, LINK=92, CENTER=0.0016;
let alpha=1, drag=null;
function physics(){
for(let i=0;i<N;i++){
const n=GRAPH.nodes[i];
for(let j=i+1;j<N;j++){
const m=GRAPH.nodes[j];
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
}
}
for(const e of edges){
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
}
for(const n of GRAPH.nodes){
if(n===drag){n.vx=n.vy=0;continue;}
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
}
alpha*=0.985;
}
function loop(){step();draw();requestAnimationFrame(loop);}loop();
// --- camera / fit-to-view -------------------------------------------------
let cam={x:0,y:0,k:1};
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
addEventListener('mouseup',()=>{drag=null;pan=null;});
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
function fitView(){
if(!N){return;}
let a=1e9,b=1e9,c=-1e9,d=-1e9;
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
}
let fitted=false;
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
// --- focus / hover state --------------------------------------------------
let hoverNode=null, focusNode=null;
function active(){return focusNode||hoverNode;}
function neighbors(n){return n?adj[n.id]:null;}
function draw(){
ctx.setTransform(DPR,0,0,DPR,0,0);
ctx.clearRect(0,0,W,H);
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
const [er,eg,eb]=PAL.edge;
// edges: clearly visible by default; when a node is focused, only its edges
// stay bright and the rest fade back (but never to invisible).
for(const e of edges){
const lit = act && (e.s===act||e.t===act);
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
// never thin out to a hairline when zoomed in or on dense graphs.
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
ctx.lineWidth=Math.max(1,screenW)/cam.k;
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
}
// nodes
for(const n of GRAPH.nodes){
const dim = act && n!==act && !(nb&&nb.has(n.id));
ctx.globalAlpha = dim?0.22:1;
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
ctx.globalAlpha=1;
}
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
ctx.textBaseline='middle';
for(const n of GRAPH.nodes){
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
if(!show) continue;
const txt=(n.label||'').slice(0,20);
if(!txt) continue;
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
const tx=n.x+n.r+4/cam.k, ty=n.y;
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
ctx.strokeText(txt,tx,ty);
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
}
}
function loop(){
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
draw();
requestAnimationFrame(loop);
}
// Pre-settle a few iterations so the first paint is already roughly laid out.
for(let i=0;i<60;i++)physics();
fitView();
loop();
// --- picking & interaction ------------------------------------------------
function pick(p){
let best=null,bd=1e9;
for(const n of GRAPH.nodes){
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
if(d<hit*hit && d<bd){bd=d;best=n;}
}
return best;
}
let pan=null, downXY=null, moved=false;
canvas.addEventListener('mousedown',ev=>{
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
canvas.classList.add('grabbing');
});
addEventListener('mousemove',ev=>{
const p=toWorld(ev.clientX,ev.clientY);
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
hoverNode=drag||pan?hoverNode:pick(p);
if(hoverNode && !drag && !pan){
tip.style.display='block';
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
tip.style.top=(ev.clientY+14)+'px';
tip.textContent=hoverNode.label||'';
} else { tip.style.display='none'; }
});
addEventListener('mouseup',ev=>{
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
renderPanel();
}
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
});
canvas.addEventListener('wheel',ev=>{
ev.preventDefault();
const s=ev.deltaY<0?1.12:0.89;
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
},{passive:false});
// --- detail panel ---------------------------------------------------------
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
function renderPanel(){
if(!focusNode){panel.style.display='none';return;}
const n=focusNode;
panel.querySelector('h3').textContent=n.label||n.id;
const cat=n.category?' · '+n.category:'';
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
const nb=panel.querySelector('.nb'); nb.innerHTML='';
let count=0;
for(const id of adj[n.id]){
if(count++>=24)break;
const m=idx[id]; if(!m)continue;
const s=document.createElement('span'); s.textContent=m.label||id;
s.title=m.label||id;
s.style.cursor='pointer';
s.onclick=()=>{focusNode=m;renderPanel();};
nb.appendChild(s);
}
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
panel.style.display='block';
}
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
</script>
</body>
</html>"""
@@ -84,6 +84,46 @@ references(外部来源汇总)、decision(架构/设计决策)、journal
}"""
_DOC_SYSTEM_PROMPT = """\
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
**陈述式知识(declarative knowledge)**,而不是照抄原文。
要求:
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
- 用自己的话重组为结构化知识,不要逐段复制原文。
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
- 用中文输出。
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
- 文档没有可沉淀的知识时返回空的 notes 数组。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"notes": [
{
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "文档背景或适用场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}
]
}"""
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
parts: list[str] = []
for i, t in enumerate(turns, 1):
@@ -171,21 +211,27 @@ def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
def _build_note_draft(
note_data: dict[str, Any],
*,
thread_id: str,
thread_title: str | None,
turns: list[dict[str, str]],
source_type: str,
source_id: str | None,
source_key: str,
fallback_title: str,
title_override: str | None,
tags: list[str] | None,
project_name: str | None,
sources: list[SourceDraft],
source_key_suffix: str = "",
) -> NoteDraft:
"""Render one note dict into a :class:`NoteDraft`."""
"""Render one note dict into a :class:`NoteDraft`.
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
``source_key`` and a ``fallback_title`` (used when neither the user override
nor the LLM produced a usable title), so both thread and document capture
share one renderer.
"""
category = str(note_data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
summary = str(note_data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(note_data, final_title, related, sources)
@@ -210,15 +256,15 @@ def _build_note_draft(
summary=summary or final_title,
content_md=body,
category=category,
source_type="thread",
source_id=thread_id,
source_type=source_type,
source_id=source_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=f"thread:{thread_id}{source_key_suffix}",
source_key=source_key,
)
@@ -293,6 +339,7 @@ async def extract_thread_llm_multi(
else:
note_dicts = note_dicts[: max(1, max_notes)]
fallback_title = extractor._auto_title(thread_title, turns)
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
# The user-supplied title override only applies to a single-note result;
@@ -302,14 +349,100 @@ async def extract_thread_llm_multi(
drafts.append(
_build_note_draft(
nd,
thread_id=thread_id,
thread_title=thread_title,
turns=turns,
source_type="thread",
source_id=thread_id,
source_key=f"thread:{thread_id}{suffix}",
fallback_title=fallback_title,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
)
)
return drafts
async def extract_document_llm_multi(
text: str,
*,
doc_id: str,
doc_title: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
model_name: str | None = None,
max_input_chars: int = 12000,
allow_multi: bool = True,
max_notes: int = 4,
) -> list[NoteDraft]:
"""Distill an imported **document** into one *or several* knowledge notes.
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
when the LLM path fails (callers fall back to a verbatim note); raises
:class:`LLMRejectedIngest` when the model deliberately judges the document
worthless. Resulting notes carry ``source_type="file"``.
"""
text = (text or "").strip()
if not text:
return []
sources = sources or []
try:
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
prompt = [
SystemMessage(content=_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
]
response = await model.ainvoke(prompt)
raw = response.content if isinstance(response.content, str) else str(response.content)
data = _parse_json(raw)
except Exception:
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
return []
if not data:
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
return []
note_dicts = _notes_from_data(data)
if not note_dicts:
reason = str(data.get("reason") or "")[:200]
if _is_falsey(data.get("should_ingest")):
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
raise LLMRejectedIngest(reason or "no knowledge value")
logger.warning(
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
doc_id,
data.get("should_ingest"),
reason,
)
return []
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
override = title if (len(note_dicts) == 1) else None
suffix = "" if len(note_dicts) == 1 else f"#{i}"
drafts.append(
_build_note_draft(
nd,
source_type="file",
source_id=doc_id,
source_key=f"file:{doc_id}{suffix}",
fallback_title=fallback_title,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
source_key_suffix=suffix,
)
)
return drafts
@@ -394,6 +394,129 @@ class KnowledgeService:
)
return await self._persist_draft(draft, created_by=created_by)
def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft:
"""Build a verbatim fallback note from a file when LLM distillation fails.
Lands as a ``draft`` (like the thread fallback) since it is the raw
document, not distilled knowledge.
"""
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
name = (title or Path(filename).stem or "文档知识").strip() or "文档知识"
summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280]
return NoteDraft(
title=name,
summary=summary,
content_md=body,
category="references",
source_type="file",
source_id=doc_id,
status="draft",
tags=base_tags,
confidence=0.4,
related=extractor.related_links(self.project_name),
source_key=f"file:{doc_id}",
)
async def capture_file(
self,
*,
content_md: str,
filename: str,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
model_name: str | None = None,
skip_existing: bool = False,
) -> dict[str, Any]:
"""Import a file (already converted to Markdown/text) into knowledge notes.
Runs the same wiki-capture LLM distillation as thread capture, so an
uploaded document becomes one or more declarative wiki notes in the vault.
Falls back to a verbatim ``draft`` note when the LLM is disabled or returns
nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``;
raises :class:`EmptyThreadError` when there is nothing to keep.
"""
import hashlib
text = (content_md or "").strip()
if not text:
raise EmptyThreadError(f"File {filename} is empty")
# Strip any frontmatter the document carries; the vault writer adds its own.
_, parsed = MarkdownVault.parse(text)
body = (parsed or text).strip()
if not body:
raise EmptyThreadError(f"File {filename} has no body content")
# Stable id from name + size so re-importing the same file can be skipped.
doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16]
logger.info(
"[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)",
filename,
len(body),
doc_id,
self.config.llm_extract_enabled,
skip_existing,
)
if skip_existing:
existing = await self.repo.find_by_source("file", doc_id)
if existing:
logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id"))
return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False}
drafts: list[NoteDraft] = []
llm_fallback = False
if self.config.llm_extract_enabled:
try:
drafts = await llm_extractor.extract_document_llm_multi(
body,
doc_id=doc_id,
doc_title=title or Path(filename).stem,
title=title,
tags=tags,
project_name=self.project_name,
model_name=model_name or self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
allow_multi=self.config.multi_note_enabled,
)
except llm_extractor.LLMRejectedIngest as exc:
raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc
if not drafts:
llm_fallback = True
if not drafts:
drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)]
effective_status = status
if llm_fallback and normalize_status(status) == "approved":
effective_status = "draft"
normalized = normalize_status(effective_status)
for draft in drafts:
draft.status = normalized
# Cross-link sibling notes split from the same document (resolved by title
# once all are persisted), mirroring multi-note thread capture.
if len(drafts) > 1:
for i, draft in enumerate(drafts):
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
if siblings:
draft.content_md = _inject_related_notes(draft.content_md, siblings)
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
logger.info(
"[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s",
filename,
len(notes),
", ".join(n.get("title", "") for n in notes),
normalized,
llm_fallback,
)
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
# -- CRUD -------------------------------------------------------------
async def list_notes(
self,
@@ -131,6 +131,8 @@ class AgentStore(ABC):
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
@@ -61,6 +61,8 @@ class MemoryAgentStore(AgentStore):
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"featured_order": None,
"created_at": now,
"updated_at": now,
@@ -93,7 +95,7 @@ class MemoryAgentStore(AgentStore):
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
@@ -103,7 +105,7 @@ class MemoryAgentStore(AgentStore):
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
@@ -159,7 +161,7 @@ class MemoryAgentStore(AgentStore):
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
@@ -198,16 +200,27 @@ class MemoryAgentStore(AgentStore):
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
@@ -27,6 +27,12 @@ class AgentRow(Base):
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# Publish square (发布广场) this agent is filed under. A pure organizational
# category with no access control — independent of ``published`` /
# ``is_private``. Empty string = "not assigned" → the listing layer treats
# it as the admin-configured default square. Each agent belongs to at most
# one square (single column), so it can never be published to two squares.
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
@@ -37,7 +43,7 @@ class AgentRow(Base):
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class AgentFavoriteRow(Base):
class AgentFavoriteRow(Base):
"""A user's bookmark of a (typically public-square) agent.
Lets a user collect agents from the 广场 (square) into their "我的" tab
@@ -51,22 +57,22 @@ class AgentFavoriteRow(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentPinRow(Base):
"""A per-user pin that keeps an agent near the top of that user's lists."""
__tablename__ = "agent_pins"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentSkillRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentPinRow(Base):
"""A per-user pin that keeps an agent near the top of that user's lists."""
__tablename__ = "agent_pins"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentSkillRow(Base):
"""Denormalized cache of a custom agent's selected skills.
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
@@ -93,6 +93,7 @@ class AgentRepository(AgentStore):
user_id=data.get("user_id"),
published=bool(data.get("published", False)),
is_private=bool(data.get("is_private", False)),
square_id=str(data.get("square_id") or ""),
created_at=now,
updated_at=now,
)
@@ -131,7 +132,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id != user_id:
return None
for key in ("name", "description", "published", "is_private"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -144,7 +145,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id is not None:
return None
for key in ("name", "description", "published", "is_private"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -224,7 +225,7 @@ class AgentRepository(AgentStore):
row = await session.get(AgentRow, agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private"):
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
@@ -290,6 +291,8 @@ class AgentRepository(AgentStore):
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
@@ -302,6 +305,14 @@ class AgentRepository(AgentStore):
if owner and owner.strip():
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
filters.append(AgentRow.user_id == owner.strip())
if square_id and square_id.strip():
sid = square_id.strip()
if square_is_default:
# Legacy / unassigned rows (square_id == "") belong to the
# default square, so include them alongside explicit matches.
filters.append(or_(AgentRow.square_id == sid, AgentRow.square_id == ""))
else:
filters.append(AgentRow.square_id == sid)
count_stmt = select(func.count()).select_from(AgentRow)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
@@ -0,0 +1,74 @@
"""Add ``agents.square_id`` and ``skills.square_id`` for the publish-square (发布广场) feature.
A nullable-default (server_default '') string column tagging which admin-
configured publish square an agent / skill is filed under. Pure organizational
category — independent of the ``published`` / ``is_private`` flags. Empty string
means "unassigned" and is resolved to the default square at read time.
Revision ID: 20260611_01
Revises: 20260609_02
Create Date: 2026-06-11
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260611_01"
down_revision = "20260609_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def _add_square_id(table_name: str, index_name: str) -> None:
if _has_table(table_name) and not _has_column(table_name, "square_id"):
op.add_column(
table_name,
sa.Column(
"square_id",
sa.String(length=64),
nullable=False,
server_default="",
),
)
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, ["square_id"])
def _drop_square_id(table_name: str, index_name: str) -> None:
if _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
if _has_column(table_name, "square_id"):
op.drop_column(table_name, "square_id")
def upgrade() -> None:
_add_square_id("agents", "ix_agents_square_id")
_add_square_id("skills", "ix_skills_square_id")
def downgrade() -> None:
_drop_square_id("skills", "ix_skills_square_id")
_drop_square_id("agents", "ix_agents_square_id")
@@ -71,6 +71,7 @@ class MemorySkillStore(SkillStore):
"published": bool(data.get("published", False)),
"detail": data.get("detail"),
"name_zh": data.get("name_zh"),
"square_id": str(data.get("square_id") or ""),
"created_at": _now(),
"updated_at": _now(),
}
@@ -106,6 +107,8 @@ class MemorySkillStore(SkillStore):
record["detail"] = data["detail"]
if "name_zh" in data:
record["name_zh"] = data["name_zh"]
if "square_id" in data:
record["square_id"] = str(data["square_id"] or "")
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -121,6 +124,8 @@ class MemorySkillStore(SkillStore):
record["detail"] = data["detail"]
if "name_zh" in data:
record["name_zh"] = data["name_zh"]
if "square_id" in data:
record["square_id"] = str(data["square_id"] or "")
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -33,6 +33,10 @@ class SkillRow(Base):
# the on-disk SKILL.md) so it can be set for built-in skills too, and so the
# shared source file is never rewritten.
name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
# Publish square (发布广场) this skill is filed under once published. Pure
# organizational category, no access control. Empty string = unassigned →
# the listing layer resolves it to the default square. One square per skill.
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -65,6 +65,7 @@ class SkillRepository(SkillStore):
published=bool(data.get("published", False)),
detail=data.get("detail"),
name_zh=data.get("name_zh"),
square_id=str(data.get("square_id") or ""),
created_at=now,
updated_at=now,
)
@@ -109,6 +110,8 @@ class SkillRepository(SkillStore):
row.detail = data["detail"]
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "square_id" in data:
row.square_id = str(data["square_id"] or "")
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -127,6 +130,8 @@ class SkillRepository(SkillStore):
row.detail = data["detail"]
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "square_id" in data:
row.square_id = str(data["square_id"] or "")
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -0,0 +1,376 @@
"""Post-turn background skill review worker.
After a conversation turn completes, SkillReviewMiddleware calls
``maybe_start_review()``. This module runs a lightweight LLM React agent
(with skill_manage + skill_list tools) that reads the conversation history
and decides whether to create or patch skills.
Concurrency model: per-thread asyncio.Lock + a single _pending slot.
If a review is already running for the thread, the newer messages overwrite
the pending slot; once the current review finishes, it drains that slot.
"""
from __future__ import annotations
import asyncio
import logging
import textwrap
from typing import TYPE_CHECKING, Any
from langchain_core.messages import BaseMessage
if TYPE_CHECKING:
from deerflow.config.app_config import AppConfig
logger = logging.getLogger(__name__)
_locks: dict[str, asyncio.Lock] = {}
_pending: dict[str, list[BaseMessage]] = {}
def _get_lock(thread_id: str) -> asyncio.Lock:
if thread_id not in _locks:
_locks[thread_id] = asyncio.Lock()
return _locks[thread_id]
def extract_tool_call_count(messages: list[BaseMessage]) -> int:
"""Count total tool calls across all AI messages in the conversation."""
count = 0
for msg in messages:
tool_calls: Any = None
if hasattr(msg, "tool_calls"):
tool_calls = msg.tool_calls
elif isinstance(msg, dict):
tool_calls = msg.get("tool_calls")
if tool_calls:
count += len(tool_calls)
return count
def _format_messages_for_review(messages: list[BaseMessage], max_messages: int) -> str:
"""Serialize recent messages to a readable text block for the LLM reviewer."""
recent = messages[-max_messages:]
lines: list[str] = []
for msg in recent:
role = type(msg).__name__.replace("Message", "")
content = msg.content if isinstance(msg.content, str) else str(msg.content)
tool_calls = getattr(msg, "tool_calls", None)
if tool_calls:
calls_str = ", ".join(
f"{tc.name if hasattr(tc, 'name') else tc.get('name', '?')}()"
for tc in tool_calls
)
lines.append(f"[{role}] <tool_calls: {calls_str}>")
else:
preview = content[:400].replace("\n", " ")
lines.append(f"[{role}] {preview}")
return "\n".join(lines)
_REVIEW_SYSTEM_PROMPT = textwrap.dedent("""\
# Role: 技能评审代理(保守模式)
## Profile
- language: zh-CN
- description: 审阅已完成对话,仅在出现强证据的高价值可复用工作流时才沉淀为技能。**默认决策是 do-nothing** —— 宁可漏掉一个普通技能,也绝不创建垃圾技能。
- background: 历史问题:之前版本触发过于宽松,沉淀了大量低价值技能,污染了技能库。本版本针对性收紧。
- personality: 极度保守、苛刻挑剔、宁缺勿滥;面对模糊场景默认 do-nothing。
- expertise: 高价值工作流识别、严格去重、SKILL.md 规范、do-nothing 判定。
- target_audience: 技能库维护者、平台工程师。
## Skills
1. 工作流抽象与评审
- 价值评估: 区分一次性数据请求与具有沉淀价值的工作流。
- 去数据化归纳: 抽象出与具体数据无关的通用步骤、参数与注意事项。
- 决策映射: 在 do-nothing / patch / create 三者中做出可解释的唯一决定。
2. 文档与规范控制
- SKILL.md 编写: 简洁步骤 + 命令 + 坑点提示(非对话稿)。
- 命名规范: kebab-case(小写字母/数字/连字符,≤64 字符)。
- 触发描述: 以 "When" 开头的单句(≤1024 字符、无尖括号)。
- 结构校验: frontmatter 允许键、name 一致性、首尾分隔线。
3. 工具使用与错误处理
- skill_list 优先调用,识别功能重叠与现有覆盖范围。
- skill_manage 正确选 action=create / patch,并传递合规内容。
- 报错诊断与修正后才能重试,禁止内容不变重提。
4. 质量控制与安全
- 去重策略: 少而精,避免噪声与重复。
- 隐私合规: 严禁嵌入用户特定数据、文件路径、凭据。
- 一致性: name / description / content 三者语义对齐。
## Rules
### 0. 默认决策
**默认就是 do-nothing。** 仅当下面"必要条件"全部满足时才考虑创建;任一不满足 → do-nothing。垃圾技能比少一个技能更糟糕。
### 1. 创建必要条件(必须全部满足,AND 关系)
要创建一个新技能,以下条件**必须同时**满足:
1. **多步且有依赖**: 对话中至少 5 次工具调用,且步骤间存在依赖关系(不是并行查询,也不是同一工具的简单循环)。
2. **包含纠错经验**: 包含至少一次失败 / 报错 / 意外结果,agent 通过修正后达到目标。这种"踩坑—纠错"才是有沉淀价值的知识。
3. **跨场景可复用**: 抽离掉对话中的具体数据后,步骤本身在不同输入 / 不同数据集下仍然成立。
4. **不与现有技能重叠**: skill_list 中没有功能覆盖率 ≥70% 的现有技能(若有,应当 patch 而非新建)。
5. **能写出有料的 SKILL.md**: 抽象后能产出 ≥300 字、包含具体步骤 / 命令 / 坑点的工作流指南。写不出说明本来就没料。
### 2. 明确不要创建(即使工具调用很多也不要建)
- **单工具循环**: 连续多次 web_search 都只是搜索,没有形成工作流。
- **一次性数据查询**: 仅是"搜索 + 总结"模式,没有可复用结构。
- **LLM 常识可答**: 如"如何配置 Python 虚拟环境"这类公开文档 / 通识问题。
- **流程过度通用**: 如"先搜索再总结再回答"这种描述等于没说。
- **场景过窄**: 只在某一特定数据集 / 系统 / 项目下有效,换环境就失效。
- **内容字数不足**: 写不出 ≥300 字含真干货的 SKILL.md。
- **仅为"做点什么"而建**: 找不到清晰理由就 do-nothing,不要为了"完成 review"而强行创建。
### 3. 创建前自检(4 题,全部"是"才能 create)
提交 create 前在心中依次回答以下问题,任一为"否" → 立即放弃创建、改 do-nothing:
1. 如果我不创建这个技能,下一个遇到类似任务的用户会损失什么?(答得上来才证明有沉淀价值)
2. 这个技能是否能被一句话简短复述?若"能",说明信息量不足 → do-nothing。
3. SKILL.md 写完后是否包含至少 3 个具体步骤或命令?
4. 步骤中是否含有非显而易见的坑点提示(不是 LLM 凭常识就能猜到的)?
### 4. 单次 review 上限
**每次 review 最多创建 1 条新技能。** 即便发现多个候选,也只挑最有价值的那一个,其余 do-nothing。这是硬约束,不接受例外。
### 5. Patch 优先于 Create
若 skill_list 中已存在功能相近(即便不完全相同)的技能,优先选择 patch 现有技能补充改进点,而非新建。新技能名额只留给**全新类别**的工作流。
### 6. 格式与命名硬约束
- 命名:kebab-case,仅含小写字母 / 数字 / 连字符,≤64 字符(如 git-bisect-bug-hunt、api-load-test)。
- description:以 "When " 开头,单句,≤1024 字符,禁止尖括号 < >。
- content:合法 SKILL.md,包含 frontmatter 与正文。
- frontmatter 仅允许键:name、description、license、allowed-tools、metadata、compatibility、version、author。
- frontmatter 首行与末行必须为 "---" 独占一行。
- frontmatter 的 name 字段必须与 skill_manage 调用的 name 参数**完全一致**;不一致直接调用失败。
- 禁止添加 tags、category、type 等额外键。
- content 必须是简洁工作流指南,禁止对话转录。
- 严禁包含用户特定数据、文件路径、凭据。
### 7. 错误处理
- 调用 skill_manage 报错 → 阅读错误信息,定位根因,**修正内容后**再重试。
- 同名已存在 → 改用 action=patch(直接 create 同名会失败)。
- 禁止在未修改内容的情况下重复提交。
## Workflows
### 目标
仅在发现**高价值、可复用、有踩坑沉淀**的工作流时,创建或修补一条技能;其余一律 do-nothing。
### 步骤 1: 获取基线
调用 skill_list,记录现有技能名称与功能覆盖范围。这一步在每次 review 中必须执行。
### 步骤 2: 价值初筛(三关,任一不过即 do-nothing 结束)
通读对话并依次确认:
- 工具调用总数是否 ≥5?否则 do-nothing 结束。
- 是否包含失败 / 报错 / 修正过程?否则 do-nothing 结束。
- 步骤间是否存在依赖关系?否则 do-nothing 结束。
只有三关全过才进入步骤 3。
### 步骤 3: 候选抽象
将对话抽象为候选技能要素:
- 候选名称(kebab-case)
- 触发描述(When ... )
- 关键步骤、命令、坑点提示
- 剔除所有数据特定内容与隐私信息
### 步骤 4: 重叠检查
与 skill_list 比对:
- 完全无重叠 → 候选 action=create,进入步骤 5。
- 部分重叠(功能覆盖 ≥70%)→ 候选 action=patch,跳到步骤 6。
- 完全覆盖 → do-nothing 结束。
### 步骤 5: 创建前自检
若候选为 create,依次回答 Rules 第 3 节的 4 个自检问题。任一为"否" → 立即放弃创建,do-nothing 结束。全部为"是"才进入步骤 6。
### 步骤 6: 提交
- create:调用 skill_manage(action="create", name, content)。
- patch:调用 skill_manage(action="patch", name, content),仅针对差异 / 改进点。
- 一次 review 最多一次写操作。
### 步骤 7: 错误处理
若报错,按 Rules 第 7 节处理:阅读错误信息 → 修正根因 → 再试。禁止内容不变重提。
### 预期结果
- **多数情况下**: do-nothing。这是好的结果,不是失败。
- **少数情况下**: 一条高价值新技能,或对现有技能的一次有针对性 patch。
- **永远不要**: 为了"做点什么"而创建低价值技能。
## Initialization
作为技能评审代理(保守模式),你必须严格遵守 Rules,按 Workflows 执行。**默认决策是 do-nothing。** 只有强证据完全满足所有创建条件时,才考虑创建。创建一条垃圾技能比放弃一条普通技能更糟糕。
""")
async def _run_review(thread_id: str, messages: list[BaseMessage], app_config: "AppConfig", user_id: str = "default") -> None:
"""Core review logic: create_agent with full middleware chain, no 'silent creation' constraint.
Uses create_agent (the deerflow agent factory) directly so we can:
- Reuse the full middleware chain (ToolErrorHandlingMiddleware, etc.)
- Reuse the full tool set
- But pass app_config with skill_evolution.enabled=False to strip the
"Never create skills silently without user confirmation" constraint
from the system prompt.
Review instructions are injected via PromptPrefixMiddleware through
additional_kwargs["prompt_prefix"] on the HumanMessage.
"""
from langchain.agents import create_agent
from langchain_core.messages import HumanMessage
from langchain_core.runnables import RunnableConfig
from deerflow.agents.lead_agent.prompt import apply_prompt_template
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
from deerflow.agents.thread_state import ThreadState
from deerflow.models import create_chat_model
from deerflow.tools.tools import get_available_tools
evolution_cfg = app_config.skill_evolution
timeout = evolution_cfg.review_worker.timeout_seconds
max_messages = evolution_cfg.review_worker.max_messages
model_name = evolution_cfg.review_worker.model_name
conversation_text = _format_messages_for_review(messages, max_messages)
# Disable skill_evolution so the system prompt does NOT include
# "Never create skills silently without user confirmation".
# Also prevents SkillReviewMiddleware from registering (no recursive reviews).
review_app_config = app_config.model_copy(deep=True)
review_app_config.skill_evolution.enabled = False
try:
model = create_chat_model(name=model_name, app_config=review_app_config)
except Exception:
logger.warning("skill_review_worker: could not load model %r, aborting review", model_name)
return
tools = get_available_tools(app_config=review_app_config, subagent_enabled=False)
middlewares = build_lead_runtime_middlewares(app_config=review_app_config)
middlewares.append(PromptPrefixMiddleware())
system_prompt = apply_prompt_template(
subagent_enabled=False,
max_concurrent_subagents=0,
app_config=review_app_config,
memory_injection_enabled=False,
)
agent = create_agent(
model=model,
tools=tools,
middleware=middlewares,
system_prompt=system_prompt,
state_schema=ThreadState,
)
# RunnableConfig supplies thread_id so ThreadDataMiddleware is satisfied.
import uuid
run_config = RunnableConfig(configurable={"thread_id": f"__skill-review__{uuid.uuid4()}"})
# Review instructions arrive as prompt_prefix: PromptPrefixMiddleware prepends
# them before every model call without persisting them to state.
first_message = HumanMessage(
content=(
"Review this conversation and apply any skill changes you deem appropriate.\n\n"
f"<conversation>\n{conversation_text}\n</conversation>"
),
additional_kwargs={"prompt_prefix": _REVIEW_SYSTEM_PROMPT},
)
def _invoke_with_user_context() -> dict:
from deerflow.runtime.user_context import set_current_user, reset_current_user
class _SimpleUser:
def __init__(self, uid: str) -> None:
self.id = uid
token = set_current_user(_SimpleUser(user_id))
try:
return agent.invoke({"messages": [first_message]}, config=run_config)
finally:
reset_current_user(token)
try:
result = await asyncio.wait_for(
asyncio.get_event_loop().run_in_executor(None, _invoke_with_user_context),
timeout=timeout,
)
# Log the final AI message so we can see what the review agent decided.
final_msgs = (result or {}).get("messages", [])
final_text = ""
for msg in reversed(final_msgs):
content = getattr(msg, "content", None)
if content and isinstance(content, str) and content.strip():
final_text = content.strip()[:1000]
break
logger.info(
"skill_review_worker: review completed for thread %s — agent said: %s",
thread_id,
final_text or "(no text response)",
)
except TimeoutError:
logger.warning("skill_review_worker: review timed out after %ds (thread %s)", timeout, thread_id)
except Exception:
logger.exception("skill_review_worker: review failed for thread %s", thread_id)
async def _run_review_with_pending(
thread_id: str,
messages: list[BaseMessage],
app_config: "AppConfig",
user_id: str = "default",
) -> None:
"""Run one review; if a newer snapshot arrived while we ran, drain it."""
lock = _get_lock(thread_id)
async with lock:
await _run_review(thread_id, messages, app_config, user_id)
pending = _pending.pop(thread_id, None)
if pending is not None:
asyncio.create_task(_run_review_with_pending(thread_id, pending, app_config, user_id))
def maybe_start_review(
thread_id: str,
messages: list[BaseMessage],
app_config: "AppConfig",
user_id: str = "default",
) -> bool:
"""Schedule a background review for ``thread_id``.
Returns True if a new task was started, False if queued or skipped.
"""
evolution_cfg = app_config.skill_evolution
if not evolution_cfg.enabled or not evolution_cfg.review_worker.enabled:
return False
tool_count = extract_tool_call_count(messages)
if tool_count < evolution_cfg.review_worker.min_tool_calls:
logger.debug(
"skill_review_worker: skipping review for thread %s (%d tool calls < min %d)",
thread_id,
tool_count,
evolution_cfg.review_worker.min_tool_calls,
)
return False
lock = _get_lock(thread_id)
if lock.locked():
_pending[thread_id] = messages
logger.debug("skill_review_worker: queued review for thread %s (review in progress)", thread_id)
return False
asyncio.create_task(_run_review_with_pending(thread_id, messages, app_config, user_id))
logger.info("skill_review_worker: started review for thread %s (%d tool calls) user=%s", thread_id, tool_count, user_id)
return True
@@ -1,7 +1,7 @@
"""笔记本 RAG 检索工具。
"""空间 RAG 检索工具。
从 LangGraph configurable 中读取 notebook_search_space_id / notebook_source_ids /
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在笔记本模式下引用回答。
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在空间模式下引用回答。
API base URL 通过环境变量 DEER_FLOW_NOTEBOOK_API_BASE 配置。
"""
@@ -134,27 +134,58 @@ def _default_agent_store() -> Any | None:
return None
async def _assert_agent_may_delete(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
"""Pre-flight ownership check before agent-delete touches the on-disk file.
async def _assert_skill_is_agent_authored(name: str) -> None:
"""Refuse to modify skills authored by a human (UI upload / adopt / legacy).
LocalSkillStorage doesn't know about users — it deletes anything that
matches by name. Without this guard, a malicious or hallucinating agent
could destroy another user's skill just by knowing the name. We refuse
unless the DB confirms ``user_id`` owns the skill outright.
Even when ownership lines up, we draw a hard line at content the *human*
curated through the Web UI: uploads, adopted orphan skills, and legacy
skills with no usage entry (which default to ``source='upload'``).
Those represent reviewed content the user has explicitly endorsed —
letting the agent rewrite them risks silent drift away from what was
approved. Only skills marked ``source='agent'`` (set by
``mark_agent_created`` in the ``create`` flow) remain editable so the
evolution loop can keep refining them.
Returns the DB record for use in the subsequent cascade step. Raises
``ValueError`` (mapped to a tool error visible to the LLM) on refusal.
Raises ``ValueError`` (mapped to a tool error visible to the LLM) on
refusal.
"""
from deerflow.skills.usage import get_entry
entry = await _to_thread(get_entry, name)
if entry.get("source") != "agent":
raise ValueError(
f"Refused: skill '{name}' was authored by a human via the Web UI "
f"and cannot be modified by an agent. Ask the owner to edit it "
f"through the Web UI, or have the user duplicate it to a new "
f"agent-owned skill first."
)
async def _assert_agent_may_modify(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
"""Pre-flight ownership check before an agent-driven write touches a custom skill.
LocalSkillStorage operates by name only — patch/edit/write_file/remove_file
and delete all resolve a name to a directory without knowing who owns it.
Without this guard a malicious or hallucinating agent could overwrite or
destroy another user's skill (including UI-uploaded skills) just by
knowing the name. We refuse unless the DB confirms ``user_id`` owns the
skill outright.
Returns the DB record so callers that need owner info (e.g. delete
cascade) can reuse it. Raises ``ValueError`` (mapped to a tool error
visible to the LLM) on refusal.
"""
record = await store.get_any(name)
if record is None:
# No DB row — could be a legacy/orphan skill. Refuse rather than
# delete blindly; admin can clean up via the Web UI.
# touch blindly; users should adopt it via /api/skills/reconcile-custom
# or an admin can clean it up via the Web UI.
raise ValueError(
f"Refused: skill '{name}' has no ownership record. Ask a human to delete it via the Web UI."
f"Refused: skill '{name}' has no ownership record. Ask a human to adopt or manage it via the Web UI."
)
owner = record.get("owner_user_id")
if owner is None:
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be deleted by an agent.")
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be modified by an agent.")
if owner != user_id:
raise ValueError(f"Refused: skill '{name}' is owned by another user.")
return record
@@ -200,6 +231,10 @@ async def _skill_manage_impl(
lock = _get_lock(name)
thread_id = _get_thread_id(runtime)
skill_storage = get_or_new_skill_storage()
# Resolved up-front so ownership checks in the write-action branches stay
# one-liners. Cheap when the DB is unconfigured (returns None).
user_id = get_effective_user_id()
skill_store = _default_skill_store()
async with lock:
if action == "create":
@@ -224,14 +259,16 @@ async def _skill_manage_impl(
# surfaces it (custom skills without a DB row are filtered out by
# ``GET /api/skills``). Resolve the user_id *here* on the main
# async stack — contextvars don't propagate into ``to_thread``.
store = _default_skill_store()
if store is not None:
await _ensure_skill_db_registered(store, name, get_effective_user_id())
if skill_store is not None:
await _ensure_skill_db_registered(skill_store, name, user_id)
await refresh_skills_system_prompt_cache_async()
return f"Created custom skill '{name}'."
if action == "edit":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if content is None:
raise ValueError("content is required for edit.")
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
@@ -250,6 +287,9 @@ async def _skill_manage_impl(
if action == "patch":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if find is None or replace is None:
raise ValueError("find and replace are required for patch.")
skill_file = skill_storage.get_custom_skill_file(name)
@@ -277,11 +317,10 @@ async def _skill_manage_impl(
# Ownership pre-check: LocalSkillStorage deletes purely by name,
# so without this an agent could blow away another user's skill
# just by knowing the name. Raise BEFORE touching the filesystem.
user_id = get_effective_user_id()
store = _default_skill_store()
record: dict[str, Any] | None = None
if store is not None:
record = await _assert_agent_may_delete(store, name, user_id)
if skill_store is not None:
record = await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
await _to_thread(
skill_storage.delete_custom_skill,
@@ -306,9 +345,9 @@ async def _skill_manage_impl(
await _to_thread(forget, name)
except Exception:
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
if store is not None:
if skill_store is not None:
try:
await store.delete(name, user_id)
await skill_store.delete(name, user_id)
except Exception:
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
agent_store = _default_agent_store()
@@ -330,6 +369,9 @@ async def _skill_manage_impl(
if action == "write_file":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if path is None or content is None:
raise ValueError("path and content are required for write_file.")
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
@@ -347,6 +389,9 @@ async def _skill_manage_impl(
if action == "remove_file":
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
if skill_store is not None:
await _assert_agent_may_modify(skill_store, name, user_id)
await _assert_skill_is_agent_authored(name)
if path is None:
raise ValueError("path is required for remove_file.")
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)