Merge branch 'ji' into dhk-1
Resolve column_additions_mysql.sql conflict: keep square_id columns from master and roundtable_drafts.task_id from dhk-1. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
commit
bcc3d617a0
142 files changed
+14953
-2383
No files matched your search
+55
-5
@@ -109,20 +109,42 @@ async def _search_notebook(
|
||||
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
|
||||
import httpx # 懒导入:仅在笔记模式下才需要
|
||||
|
||||
# 入参全 dump(token 只打长度,不打明文,防止泄漏到日志)
|
||||
logger.info(
|
||||
"[notebook-search] called keywords=%s search_space_id=%s source_ids=%s token_len=%d",
|
||||
keywords,
|
||||
search_space_id,
|
||||
source_ids,
|
||||
len(token) if token else 0,
|
||||
)
|
||||
|
||||
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
|
||||
if not base:
|
||||
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
|
||||
# 改成 error 级别,warning 在很多部署里默认会被过滤掉看不到
|
||||
logger.error(
|
||||
"[notebook-search] DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search. "
|
||||
"请在后端启动环境里设这个环境变量(指向笔记后端根地址,例如 http://localhost:8003)。"
|
||||
)
|
||||
return []
|
||||
|
||||
full_url = f"{base}{_NOTEBOOK_SEARCH_PATH}"
|
||||
logger.info("[notebook-search] base=%s full_url=%s", base, full_url)
|
||||
|
||||
results: list[Material] = []
|
||||
mat_idx = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
|
||||
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
|
||||
"query": kw,
|
||||
"source_ids": source_ids,
|
||||
"search_space_id": search_space_id,
|
||||
"top_k": _NOTEBOOK_TOP_K,
|
||||
})
|
||||
try:
|
||||
# /mentioned-documents/search 接口无需鉴权,不传 Authorization header
|
||||
resp = await client.post(
|
||||
f"{base}{_NOTEBOOK_SEARCH_PATH}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
full_url,
|
||||
json={
|
||||
"query": kw,
|
||||
"source_ids": source_ids,
|
||||
@@ -130,10 +152,29 @@ async def _search_notebook(
|
||||
"top_k": _NOTEBOOK_TOP_K,
|
||||
},
|
||||
)
|
||||
logger.info(
|
||||
"[notebook-search] kw=%r http_status=%d body_len=%d",
|
||||
kw,
|
||||
resp.status_code,
|
||||
len(resp.content or b""),
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
# 把错误响应体打出来,方便定位 401 / 422 等
|
||||
logger.error(
|
||||
"[notebook-search] kw=%r HTTP %d response: %s",
|
||||
kw,
|
||||
resp.status_code,
|
||||
(resp.text or "")[:1000],
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
# 解析 documents 列表,每个 chunk 作为独立素材
|
||||
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
|
||||
logger.info(
|
||||
"[notebook-search] kw=%r parsed documents=%d",
|
||||
kw,
|
||||
len(documents or []),
|
||||
)
|
||||
for doc in documents or []:
|
||||
doc_title: str = doc.get("title") or "笔记文档"
|
||||
doc_url: str = doc.get("url") or ""
|
||||
@@ -154,10 +195,11 @@ async def _search_notebook(
|
||||
url=doc_url,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning("notebook search kw=%r failed: %s", kw, e)
|
||||
logger.warning("[notebook-search] kw=%r failed: %s", kw, e, exc_info=True)
|
||||
except Exception as e:
|
||||
logger.warning("notebook search client error: %s", e)
|
||||
logger.warning("[notebook-search] client error: %s", e, exc_info=True)
|
||||
|
||||
logger.info("[notebook-search] done — returning %d materials", len(results))
|
||||
return results
|
||||
|
||||
|
||||
@@ -250,6 +292,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
|
||||
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
|
||||
notebook_materials: list[Material] = []
|
||||
|
||||
if notebook_search_space_id:
|
||||
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
|
||||
notebook_token: str = cfg.get("notebook_token") or ""
|
||||
@@ -369,6 +412,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
|
||||
all_results_nested: list = []
|
||||
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
|
||||
elif material_source == "notebook" and notebook_search_space_id:
|
||||
# ── 纯笔记模式 ──
|
||||
# 用户在表单选了「我的空间」作为素材来源 → 跳过 web/intranet 检索,
|
||||
# 仅使用 Step 1.5 已拉取的 notebook_materials。
|
||||
# 与 UI 文案"从空间中检索素材"对齐。
|
||||
_emit(f"仅使用笔记素材,跳过 web/intranet 检索({len(notebook_materials)} 条)")
|
||||
all_results_nested = []
|
||||
else:
|
||||
# ── 常规 web/intranet 模式 ──
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
+39
-7
@@ -20,7 +20,7 @@ from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddlew
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
|
||||
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||||
from deerflow.config.agents_config import load_agent_config, load_agent_soul, validate_agent_id
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
@@ -331,6 +331,13 @@ def _build_middlewares(
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents))
|
||||
|
||||
# SkillReviewMiddleware — trigger background skill review after each turn.
|
||||
# Gated by skill_evolution.enabled + skill_evolution.review_worker.enabled.
|
||||
if resolved_app_config.skill_evolution.enabled and resolved_app_config.skill_evolution.review_worker.enabled:
|
||||
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
|
||||
|
||||
middlewares.append(SkillReviewMiddleware())
|
||||
|
||||
# LoopDetectionMiddleware — detect and break repetitive tool call loops
|
||||
middlewares.append(LoopDetectionMiddleware())
|
||||
|
||||
@@ -535,11 +542,30 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
|
||||
default_tools = [notebook_search_tool] + default_tools
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
|
||||
tools=default_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
system_prompt=apply_prompt_template(
|
||||
# Custom agents are steered ONLY by their own SOUL.md — no generic
|
||||
# super-agent system template and no personal memory injection. We branch to
|
||||
# the agent-only prompt builder whenever a custom agent config carries a
|
||||
# non-empty SOUL.md. The plain default lead agent (no agent_config) and
|
||||
# notebook mode keep the full template. See apply_agent_only_prompt_template.
|
||||
has_agent_soul = bool(agent_config) and bool(load_agent_soul(agent_id))
|
||||
use_agent_only_prompt = has_agent_soul and not is_notebook_mode
|
||||
|
||||
if use_agent_only_prompt:
|
||||
from deerflow.agents.lead_agent.prompt import apply_agent_only_prompt_template
|
||||
|
||||
system_prompt = apply_agent_only_prompt_template(
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_display_name,
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
writing_mode=writing_mode,
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
)
|
||||
else:
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
agent_id=agent_id,
|
||||
@@ -552,6 +578,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
),
|
||||
)
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
|
||||
tools=default_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+138
-22
@@ -615,6 +615,127 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
|
||||
"""
|
||||
|
||||
|
||||
# Minimal system prompt for custom agents that should be steered ONLY by their
|
||||
# own SOUL.md — no generic "super agent" persona scaffolding (role line, thinking
|
||||
# style, clarification system, citation rules, response-style / critical-reminder
|
||||
# defaults) and no personal memory injection. Only the mechanical sections an
|
||||
# agent still needs to actually use its configured skills/tools are kept.
|
||||
AGENT_ONLY_PROMPT_TEMPLATE = """{soul}
|
||||
{network_policy_section}{skills_section}
|
||||
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
|
||||
{subagent_section}
|
||||
|
||||
<working_directory existed="true">
|
||||
- User uploads: `/mnt/user-data/uploads` - Files uploaded by the user (automatically listed in context)
|
||||
- User workspace: `/mnt/user-data/workspace` - Working directory for temporary files
|
||||
- Output files: `/mnt/user-data/outputs` - Final deliverables must be saved here
|
||||
|
||||
**File Management:**
|
||||
- Uploaded files are automatically listed in the <uploaded_files> section before each request
|
||||
- Use `read_file` tool to read uploaded files using their paths from the list
|
||||
- For PDF, PPT, Excel, and Word files, converted Markdown versions (*.md) are available alongside originals
|
||||
- Treat `/mnt/user-data/workspace` as your default current working directory for coding and file-editing tasks
|
||||
- Final deliverables must be copied to `/mnt/user-data/outputs` and presented using `present_files` tool
|
||||
{acp_section}
|
||||
</working_directory>
|
||||
"""
|
||||
|
||||
|
||||
def apply_agent_only_prompt_template(
|
||||
*,
|
||||
agent_id: str | None,
|
||||
agent_name: str | None,
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
rag_mode_enabled: bool = False,
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
) -> str:
|
||||
"""Build a custom agent's system prompt from ONLY its SOUL.md plus the
|
||||
mechanical sections required for its configured skills/tools to work.
|
||||
|
||||
Unlike :func:`apply_prompt_template`, this deliberately omits:
|
||||
- the generic ``<role>`` "internal super agent" framing,
|
||||
- ``<thinking_style>`` / ``<clarification_system>`` / ``<response_style>`` /
|
||||
``<citations>`` / ``<critical_reminders>`` behavioural defaults,
|
||||
- the personal ``<memory>`` injection (USER.md / MEMORY.md).
|
||||
|
||||
The agent's SOUL.md is therefore the sole source of persona and behaviour.
|
||||
Skills, deferred tools, the subagent orchestration section (when enabled) and
|
||||
the working-directory contract are kept because they are capability wiring,
|
||||
not persona — dropping them would silently break agents that rely on skills
|
||||
or file output.
|
||||
"""
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
soul = get_agent_soul(storage_agent_id)
|
||||
|
||||
skills_section = get_skills_prompt_section(
|
||||
available_skills,
|
||||
app_config=app_config,
|
||||
display_enabled=rag_mode_enabled,
|
||||
)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
|
||||
n = max_concurrent_subagents
|
||||
subagent_section = _build_subagent_section(n, app_config=app_config) if subagent_enabled else ""
|
||||
|
||||
current_writing_artifact = (
|
||||
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
|
||||
if writing_artifact_path
|
||||
else ""
|
||||
)
|
||||
writing_mode_section = (
|
||||
f"""<writing_mode>
|
||||
Writing mode is enabled for this run.
|
||||
- Your final deliverable MUST be a Markdown file saved under `/mnt/user-data/outputs/` (e.g. `/mnt/user-data/outputs/report.md`).
|
||||
- If a current Markdown artifact is provided below, revise that file rather than creating a new one unless the user explicitly asks for a separate document.
|
||||
- After writing the file, call `present_files` with the Markdown file path.
|
||||
{current_writing_artifact}</writing_mode>
|
||||
"""
|
||||
if writing_mode
|
||||
else ""
|
||||
)
|
||||
|
||||
acp_section = _build_acp_section(app_config=app_config)
|
||||
custom_mounts_section = _build_custom_mounts_section(app_config=app_config)
|
||||
acp_and_mounts_section = "\n".join(section for section in (acp_section, custom_mounts_section) if section)
|
||||
|
||||
network_policy_section = _build_network_policy_section()
|
||||
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering per search term, tool, or skill.
|
||||
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
|
||||
- Do not output `<skill-display .../>` in RAG/reference mode; the frontend renders the right reference panel.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
prompt = AGENT_ONLY_PROMPT_TEMPLATE.format(
|
||||
soul=soul,
|
||||
skills_section=skills_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
subagent_section=subagent_section,
|
||||
network_policy_section=network_policy_section,
|
||||
acp_section=acp_and_mounts_section,
|
||||
)
|
||||
|
||||
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
|
||||
return prompt
|
||||
|
||||
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
@@ -637,28 +758,23 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
if not config.enabled or not config.injection.enabled:
|
||||
return ""
|
||||
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name or "default",
|
||||
config=config,
|
||||
)
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name or "default",
|
||||
config=config,
|
||||
app_config=app_config,
|
||||
)
|
||||
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
|
||||
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
|
||||
memory_content = manager.build_system_prompt()
|
||||
|
||||
if not memory_content.strip():
|
||||
return ""
|
||||
|
||||
return f"""<memory>
|
||||
{memory_content}
|
||||
</memory>
|
||||
"""
|
||||
context_content = manager.build_system_prompt()
|
||||
|
||||
if not context_content.strip():
|
||||
return ""
|
||||
|
||||
return f"{context_content}\n"
|
||||
except Exception:
|
||||
logger.exception("Failed to load memory context")
|
||||
return ""
|
||||
@@ -1078,7 +1194,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
|
||||
<notebook_mode>
|
||||
## 角色
|
||||
你是一个笔记本 AI 助手,**只能**基于笔记本文档来源回答用户问题,禁止使用任何其他知识。
|
||||
你是一个空间 AI 助手,**只能**基于空间文档来源回答用户问题,禁止使用任何其他知识。
|
||||
|
||||
## 可用知识来源
|
||||
{source_line}
|
||||
@@ -1104,7 +1220,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- 未调用 `search_notebook_sources` 就直接回答
|
||||
- 引用不在检索结果中的 chunk_id
|
||||
- 省略引用标记
|
||||
- 使用笔记本来源之外的知识
|
||||
- 使用空间来源之外的知识
|
||||
</notebook_mode>"""
|
||||
|
||||
return prompt
|
||||
+20
-5
@@ -32,6 +32,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
#: 内建 provider 的固定名字。
|
||||
BUILTIN_PROVIDER_NAME = "builtin"
|
||||
KNOWLEDGE_PROVIDER_NAME = "knowledge"
|
||||
_INTERNAL_PROVIDER_NAMES = {BUILTIN_PROVIDER_NAME, KNOWLEDGE_PROVIDER_NAME}
|
||||
|
||||
|
||||
class MemoryManager:
|
||||
@@ -49,11 +51,11 @@ class MemoryManager:
|
||||
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
|
||||
第二个会被拒绝并 log 警告。返回是否注册成功。
|
||||
"""
|
||||
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
|
||||
if not is_builtin:
|
||||
is_internal = provider.name in _INTERNAL_PROVIDER_NAMES
|
||||
if not is_internal:
|
||||
if self._has_external:
|
||||
existing = next(
|
||||
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
|
||||
(p.name for p in self._providers if p.name not in _INTERNAL_PROVIDER_NAMES),
|
||||
"unknown",
|
||||
)
|
||||
logger.warning(
|
||||
@@ -220,6 +222,7 @@ def build_memory_manager(
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
config: Any = None,
|
||||
app_config: Any = None,
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
@@ -246,7 +249,9 @@ def build_memory_manager(
|
||||
manager = MemoryManager()
|
||||
base_dir = str(get_paths().base_dir)
|
||||
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
memory_enabled = bool(getattr(config, "enabled", True) and getattr(config.injection, "enabled", True))
|
||||
|
||||
if memory_enabled and getattr(config, "version", "v2") == "v1":
|
||||
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
@@ -260,7 +265,7 @@ def build_memory_manager(
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
)
|
||||
else:
|
||||
elif memory_enabled:
|
||||
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
@@ -286,11 +291,21 @@ def build_memory_manager(
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
try:
|
||||
from deerflow.agents.memory.providers.knowledge import KnowledgeProvider
|
||||
|
||||
knowledge = KnowledgeProvider(app_config=app_config)
|
||||
if knowledge.is_available():
|
||||
manager.add_provider(knowledge)
|
||||
except Exception as exc: # noqa: BLE001 - 知识库路由失败不能阻断对话
|
||||
logger.warning("加载 Knowledge provider 失败:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
base_dir=base_dir,
|
||||
app_config=app_config,
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
+4
-1
@@ -227,7 +227,10 @@ class BuiltinFileProvider(MemoryProvider):
|
||||
保持系统提示词稳定,利于 prefix cache。
|
||||
"""
|
||||
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
|
||||
return "\n\n".join(blocks)
|
||||
content = "\n\n".join(blocks)
|
||||
if not content:
|
||||
return ""
|
||||
return f"<memory>\n{content}\n</memory>"
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
|
||||
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"""Read-only provider for the mounted policy knowledge base.
|
||||
|
||||
This provider keeps the current knowledge-base behaviour: inject the lightweight
|
||||
``routing.md`` guide and let the model decide when to read concrete files. It
|
||||
does not automatically retrieve snippets and never writes to the knowledge base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _read_routing_file(host_path: str) -> str:
|
||||
path = Path(host_path)
|
||||
if not path.is_file():
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
class KnowledgeProvider(MemoryProvider):
|
||||
"""Injects the knowledge routing guide as a read-only context provider."""
|
||||
|
||||
def __init__(self, *, app_config: Any | None = None) -> None:
|
||||
self._app_config = app_config
|
||||
self._routing_host_path: Path | None = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "knowledge"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
app_config = self._app_config or kwargs.get("app_config")
|
||||
if app_config is None:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
app_config = get_app_config()
|
||||
|
||||
self._routing_host_path = None
|
||||
for mount in app_config.sandbox.mounts or []:
|
||||
if mount.container_path.rstrip("/") == "/mnt/knowledge":
|
||||
self._routing_host_path = Path(mount.host_path) / "routing.md"
|
||||
break
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
if self._routing_host_path is None:
|
||||
return ""
|
||||
|
||||
content = _read_routing_file(str(self._routing_host_path.resolve()))
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
logger.debug("Injected knowledge routing memory from %s", self._routing_host_path)
|
||||
return f"""<knowledge_routing_memory source="/mnt/knowledge/routing.md">
|
||||
{content}
|
||||
</knowledge_routing_memory>"""
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
__all__ = ["KnowledgeProvider"]
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""SkillReviewMiddleware — triggers background skill review after each agent turn.
|
||||
|
||||
Hooks into ``after_agent``: once the lead agent graph has finished a full turn,
|
||||
this middleware counts the tool calls made and, if the threshold is met, hands
|
||||
the conversation messages to the SkillReviewWorker as a background asyncio task.
|
||||
|
||||
No blocking I/O or LLM calls happen in-band — the review runs fully in the
|
||||
background and never delays the HTTP response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.skills.review_worker import maybe_start_review
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillReviewMiddlewareState(AgentState):
|
||||
"""Minimal state schema — only needs ``messages``."""
|
||||
pass
|
||||
|
||||
|
||||
class SkillReviewMiddleware(AgentMiddleware[SkillReviewMiddlewareState]):
|
||||
"""Dispatch a background skill review after every substantial agent turn.
|
||||
|
||||
'Substantial' is defined by ``skill_evolution.review_worker.min_tool_calls``
|
||||
(default 5). Turns below the threshold are silently skipped to avoid
|
||||
wasting LLM tokens on trivial exchanges.
|
||||
"""
|
||||
|
||||
state_schema = SkillReviewMiddlewareState
|
||||
|
||||
@override
|
||||
def after_agent(self, state: Any, runtime: Runtime) -> None:
|
||||
"""Fire background review; never raises — failures are logged only."""
|
||||
try:
|
||||
app_config = get_app_config()
|
||||
messages = state.get("messages", []) if hasattr(state, "get") else []
|
||||
thread_id: str = runtime.context.get("thread_id", "") if runtime.context else ""
|
||||
if not thread_id:
|
||||
return
|
||||
user_id = get_effective_user_id()
|
||||
maybe_start_review(thread_id, messages, app_config, user_id)
|
||||
except Exception:
|
||||
logger.exception("SkillReviewMiddleware.after_agent: unexpected error (ignored)")
|
||||
+193
-45
@@ -1,21 +1,41 @@
|
||||
"""Summarization middleware extensions for DeerFlow."""
|
||||
"""Summarization middleware extensions for DeerFlow.
|
||||
|
||||
Unlike LangChain's stock ``SummarizationMiddleware`` (which runs in
|
||||
``before_model`` and **persists** a destructive ``RemoveMessage(REMOVE_ALL_MESSAGES)``
|
||||
to the checkpoint — wiping the original Q&A from both the model context *and*
|
||||
the conversation the frontend renders), this middleware compresses the history
|
||||
**transiently in ``wrap_model_call``**. The full original messages stay in state
|
||||
(so the UI keeps showing the user's real conversation); only the message list
|
||||
handed to the model for a single call is replaced with ``[summary, *recent]``.
|
||||
|
||||
The generated summary is cached per-thread so we don't pay an extra LLM call on
|
||||
every model invocation: once a thread is summarized, the cached summary is reused
|
||||
until enough *new* messages accumulate to push the compressed context back over
|
||||
the trigger threshold (sticky boundary, mirroring the stock middleware's
|
||||
post-compaction behavior).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Collection
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Collection
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol, override, runtime_checkable
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import SummarizationMiddleware
|
||||
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, RemoveMessage, ToolMessage
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, ToolMessage
|
||||
from langgraph.config import get_config
|
||||
from langgraph.graph.message import REMOVE_ALL_MESSAGES
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Upper bound on the per-thread summary cache so a long-lived process serving
|
||||
# many threads can't grow it without limit. Oldest entries are evicted first.
|
||||
_SUMMARY_CACHE_MAX_THREADS = 512
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SummarizationEvent:
|
||||
@@ -95,6 +115,22 @@ class _SkillBundle:
|
||||
skill_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SummaryCacheEntry:
|
||||
"""A cached, reusable summary for one thread.
|
||||
|
||||
``summarized_ids`` are the ids of the original messages folded into
|
||||
``summary_text``. The cache is only reusable while every one of those ids is
|
||||
still present in the live message list and no AIMessage/ToolMessage pair was
|
||||
split when the boundary was drawn (``reusable``) — otherwise filtering by id
|
||||
could orphan a tool result, so we fall back to re-summarizing.
|
||||
"""
|
||||
|
||||
summarized_ids: frozenset[str]
|
||||
summary_text: str
|
||||
reusable: bool
|
||||
|
||||
|
||||
class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
|
||||
"""Summarization middleware with pre-compression hook dispatch and skill rescue."""
|
||||
|
||||
@@ -116,69 +152,181 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
|
||||
self._preserve_recent_skill_count = max(0, preserve_recent_skill_count)
|
||||
self._preserve_recent_skill_tokens = max(0, preserve_recent_skill_tokens)
|
||||
self._preserve_recent_skill_tokens_per_skill = max(0, preserve_recent_skill_tokens_per_skill)
|
||||
# Per-thread cache of the most recent summary: thread_id -> (summarized message ids, summary text).
|
||||
# Lets us reuse a summary across model calls instead of re-running the LLM every invocation.
|
||||
self._summary_cache: OrderedDict[str, _SummaryCacheEntry] = OrderedDict()
|
||||
|
||||
@override
|
||||
def before_model(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
return self._maybe_summarize(state, runtime)
|
||||
"""Disable the stock middleware's destructive ``before_model`` compaction.
|
||||
|
||||
Compression now happens transiently in :meth:`wrap_model_call`, so the
|
||||
persisted conversation (and the frontend's view of it) is never mutated.
|
||||
"""
|
||||
return None
|
||||
|
||||
@override
|
||||
async def abefore_model(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
return await self._amaybe_summarize(state, runtime)
|
||||
"""Async counterpart of :meth:`before_model` — also a no-op."""
|
||||
return None
|
||||
|
||||
def _maybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
messages = state["messages"]
|
||||
self._ensure_message_ids(messages)
|
||||
# ---- wrap_model_call: transient, display-preserving compression ----------
|
||||
|
||||
total_tokens = self.token_counter(messages)
|
||||
if not self._should_summarize(messages, total_tokens):
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
compressed = self._compressed_messages(request)
|
||||
if compressed is not None:
|
||||
request = request.override(messages=compressed)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
compressed = await self._acompressed_messages(request)
|
||||
if compressed is not None:
|
||||
request = request.override(messages=compressed)
|
||||
return await handler(request)
|
||||
|
||||
def _compressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
|
||||
"""Build the compressed message list for this call (sync summary path).
|
||||
|
||||
Returns ``None`` when no compression is needed (the request is passed
|
||||
through untouched).
|
||||
"""
|
||||
plan = self._plan_compression(request)
|
||||
if plan is None:
|
||||
return None
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(messages)
|
||||
if cutoff_index <= 0:
|
||||
return None
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
|
||||
action, payload = plan
|
||||
if action == "reuse":
|
||||
return payload # type: ignore[return-value]
|
||||
# action == "summarize"
|
||||
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
|
||||
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
|
||||
summary = self._create_summary(messages_to_summarize)
|
||||
new_messages = self._build_new_messages(summary)
|
||||
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
RemoveMessage(id=REMOVE_ALL_MESSAGES),
|
||||
*new_messages,
|
||||
*preserved_messages,
|
||||
]
|
||||
}
|
||||
async def _acompressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
|
||||
"""Async counterpart of :meth:`_compressed_messages`."""
|
||||
plan = self._plan_compression(request)
|
||||
if plan is None:
|
||||
return None
|
||||
action, payload = plan
|
||||
if action == "reuse":
|
||||
return payload # type: ignore[return-value]
|
||||
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
|
||||
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
|
||||
summary = await self._acreate_summary(messages_to_summarize)
|
||||
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
|
||||
|
||||
async def _amaybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
messages = state["messages"]
|
||||
def _plan_compression(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
) -> tuple[str, Any] | None:
|
||||
"""Decide what to do for this model call, doing **no** LLM work.
|
||||
|
||||
Returns one of:
|
||||
|
||||
- ``None`` — pass the request through unchanged (no compression needed).
|
||||
- ``("reuse", compressed_messages)`` — reuse the cached summary.
|
||||
- ``("summarize", (messages_to_summarize, preserved, thread_id))`` — the
|
||||
caller must run the (a)sync summarizer, then call
|
||||
:meth:`_finish_summarize`.
|
||||
"""
|
||||
messages = list(getattr(request, "messages", None) or [])
|
||||
if not messages:
|
||||
return None
|
||||
self._ensure_message_ids(messages)
|
||||
thread_id = self._resolve_request_thread_id(request)
|
||||
|
||||
total_tokens = self.token_counter(messages)
|
||||
if not self._should_summarize(messages, total_tokens):
|
||||
cached = self._summary_cache.get(thread_id) if thread_id else None
|
||||
present_ids = {m.id for m in messages}
|
||||
if cached is not None and cached.reusable and cached.summarized_ids <= present_ids:
|
||||
preserved = [m for m in messages if m.id not in cached.summarized_ids]
|
||||
effective = [self._summary_message(cached.summary_text), *preserved]
|
||||
if not self._should_summarize(effective, self.token_counter(effective)):
|
||||
# Refresh LRU recency and reuse without an LLM call.
|
||||
self._summary_cache.move_to_end(thread_id)
|
||||
return ("reuse", effective)
|
||||
# New messages have pushed the compressed context back over the
|
||||
# threshold — fall through and re-summarize a larger prefix.
|
||||
|
||||
if not self._should_summarize(messages, self.token_counter(messages)):
|
||||
return None
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(messages)
|
||||
if cutoff_index <= 0:
|
||||
return None
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
|
||||
summary = await self._acreate_summary(messages_to_summarize)
|
||||
new_messages = self._build_new_messages(summary)
|
||||
messages_to_summarize, preserved = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
if not messages_to_summarize:
|
||||
return None
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
RemoveMessage(id=REMOVE_ALL_MESSAGES),
|
||||
*new_messages,
|
||||
*preserved_messages,
|
||||
]
|
||||
}
|
||||
return ("summarize", (messages_to_summarize, preserved, thread_id))
|
||||
|
||||
def _finish_summarize(
|
||||
self,
|
||||
messages_to_summarize: list[AnyMessage],
|
||||
preserved: list[AnyMessage],
|
||||
summary: str,
|
||||
thread_id: str | None,
|
||||
) -> list[AnyMessage]:
|
||||
"""Assemble the compressed message list and update the per-thread cache."""
|
||||
compressed = [self._summary_message(summary), *preserved]
|
||||
|
||||
if thread_id:
|
||||
summarized_ids = frozenset(m.id for m in messages_to_summarize if m.id is not None)
|
||||
preserved_ids = {m.id for m in preserved if m.id is not None}
|
||||
# A non-empty overlap means skill rescue split an AIMessage across the
|
||||
# boundary; reconstructing ``preserved`` by id filtering later could
|
||||
# orphan a tool result, so mark the entry non-reusable (re-summarize).
|
||||
reusable = summarized_ids.isdisjoint(preserved_ids)
|
||||
self._store_cache_entry(
|
||||
thread_id,
|
||||
_SummaryCacheEntry(summarized_ids=summarized_ids, summary_text=summary, reusable=reusable),
|
||||
)
|
||||
|
||||
return compressed
|
||||
|
||||
def _store_cache_entry(self, thread_id: str, entry: _SummaryCacheEntry) -> None:
|
||||
"""Insert/refresh a cache entry, evicting the oldest thread if over capacity."""
|
||||
self._summary_cache[thread_id] = entry
|
||||
self._summary_cache.move_to_end(thread_id)
|
||||
while len(self._summary_cache) > _SUMMARY_CACHE_MAX_THREADS:
|
||||
self._summary_cache.popitem(last=False)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_request_thread_id(request: ModelRequest) -> str | None:
|
||||
"""Best-effort thread id from the model request's runtime."""
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
return _resolve_thread_id(runtime)
|
||||
|
||||
def _summary_message(self, summary: str) -> HumanMessage:
|
||||
"""Build the summary message handed to the model.
|
||||
|
||||
It is named ``summary`` and flagged ``hide_from_ui`` defensively. Because
|
||||
it only ever lives in the transient model request (never in persisted
|
||||
state), the frontend never receives it — the user keeps seeing the
|
||||
original conversation.
|
||||
"""
|
||||
return HumanMessage(
|
||||
content=f"Here is a summary of the conversation to date:\n\n{summary}",
|
||||
name="summary",
|
||||
additional_kwargs={"hide_from_ui": True, "lc_source": "summarization"},
|
||||
)
|
||||
|
||||
@override
|
||||
def _build_new_messages(self, summary: str) -> list[HumanMessage]:
|
||||
"""Override the base implementation to let the human message with the special name 'summary'.
|
||||
And this message will be ignored to display in the frontend, but still can be used as context for the model.
|
||||
"""
|
||||
return [HumanMessage(content=f"Here is a summary of the conversation to date:\n\n{summary}", name="summary")]
|
||||
"""Kept for backwards compatibility; delegates to :meth:`_summary_message`."""
|
||||
return [self._summary_message(summary)]
|
||||
|
||||
def _partition_with_skill_rescue(
|
||||
self,
|
||||
|
||||
+1
-1
@@ -107,7 +107,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
|
||||
content=last_message.content,
|
||||
id=last_message.id,
|
||||
name=last_message.name,
|
||||
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
|
||||
additional_kwargs={**last_message.additional_kwargs, "run_id": (runtime.context or {}).get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -7,7 +7,7 @@ import threading
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, model_validator
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
@@ -56,6 +56,32 @@ def validate_agent_name(name: str | None) -> str | None:
|
||||
return validate_agent_id(name)
|
||||
|
||||
|
||||
class CommonQuestion(BaseModel):
|
||||
"""A recommended question shown on the agent chat home / sidebar.
|
||||
|
||||
``title`` is the short label rendered on the question chip; ``prompt`` is the
|
||||
full message actually sent when the user clicks it. Splitting the two lets an
|
||||
admin keep the chip terse while still sending a richly-worded question.
|
||||
|
||||
Legacy configs stored questions as bare strings (content only). The
|
||||
before-validator coerces such a string into ``{title, prompt}`` so old
|
||||
config.yaml files keep loading without a migration — the title falls back to
|
||||
a truncated copy of the content.
|
||||
"""
|
||||
|
||||
title: str = ""
|
||||
prompt: str = ""
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _coerce_legacy_string(cls, value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
title = text if len(text) <= 40 else text[:40].rstrip() + "…"
|
||||
return {"title": title, "prompt": text}
|
||||
return value
|
||||
|
||||
|
||||
class AgentConfig(BaseModel):
|
||||
"""Configuration for a custom agent."""
|
||||
|
||||
@@ -69,6 +95,11 @@ class AgentConfig(BaseModel):
|
||||
# - [] (explicit empty list): no skills
|
||||
# - ["skill1", "skill2"]: load only the specified skills
|
||||
skills: list[str] | None = None
|
||||
# Frequently-asked questions ("常用问题") shown in the agent chat sidebar.
|
||||
# Display/UX metadata only — the runtime never reads these. None = none set.
|
||||
# Accepts both the legacy ``list[str]`` shape and the new ``{title, prompt}``
|
||||
# objects (see CommonQuestion).
|
||||
common_questions: list[CommonQuestion] | None = None
|
||||
|
||||
|
||||
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
|
||||
|
||||
+39
-30
@@ -1,30 +1,39 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CuratorConfig(BaseModel):
|
||||
"""Configuration for the background skill curator."""
|
||||
|
||||
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
|
||||
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
|
||||
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
|
||||
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||
|
||||
|
||||
class SkillEvolutionConfig(BaseModel):
|
||||
"""Configuration for agent-managed skill evolution."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether the agent can create and modify skills under skills/custom.",
|
||||
)
|
||||
curator: CuratorConfig = Field(
|
||||
default_factory=CuratorConfig,
|
||||
description="Configuration for the background skill curator.",
|
||||
)
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ReviewWorkerConfig(BaseModel):
|
||||
"""Configuration for the post-turn background skill review worker."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the review worker is active.")
|
||||
min_tool_calls: int = Field(default=5, description="Minimum tool calls in a turn to trigger review. Matches the prompt's '≥5 steps' hard condition so simple queries never reach the LLM reviewer.")
|
||||
max_messages: int = Field(default=60, description="Max messages passed to the review LLM.")
|
||||
model_name: str | None = Field(default=None, description="Model to use for review; defaults to primary model.")
|
||||
timeout_seconds: int = Field(default=120, description="Overall timeout for a single review run.")
|
||||
|
||||
|
||||
class CuratorConfig(BaseModel):
|
||||
"""Configuration for the background skill curator."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||
|
||||
|
||||
class SkillEvolutionConfig(BaseModel):
|
||||
"""Configuration for agent-managed skill evolution."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether the agent can create and modify skills under skills/custom.",
|
||||
)
|
||||
curator: CuratorConfig = Field(
|
||||
default_factory=CuratorConfig,
|
||||
description="Configuration for the background skill curator.",
|
||||
)
|
||||
review_worker: ReviewWorkerConfig = Field(
|
||||
default_factory=ReviewWorkerConfig,
|
||||
description="Configuration for the post-turn background skill review worker.",
|
||||
)
|
||||
+19
-2
@@ -135,8 +135,18 @@ def resolve_user_path(user_id: str) -> Path:
|
||||
return get_paths().user_dir(user_id) / _FILENAME
|
||||
|
||||
|
||||
# Only these curator sub-fields may be overridden per user.
|
||||
# All other fields (enabled, model_name, timeouts, etc.) are global-only.
|
||||
_USER_ALLOWED_CURATOR_KEYS = {"interval_hours", "stale_after_days", "archive_after_days", "model_name"}
|
||||
|
||||
|
||||
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
|
||||
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
|
||||
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid).
|
||||
|
||||
Only the three curator threshold keys are honoured; any other keys
|
||||
(e.g. legacy ``enabled`` or ``model_name``) are silently stripped so that
|
||||
old files written by previous versions cannot override global-only settings.
|
||||
"""
|
||||
path = resolve_user_path(user_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
@@ -154,7 +164,14 @@ def load_user_overrides(user_id: str) -> dict[str, Any] | None:
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
|
||||
return None
|
||||
return data
|
||||
# Strip top-level keys that are not user-configurable (e.g. legacy "enabled").
|
||||
curator_raw = data.get("curator")
|
||||
if not isinstance(curator_raw, dict):
|
||||
return None
|
||||
filtered_curator = {k: v for k, v in curator_raw.items() if k in _USER_ALLOWED_CURATOR_KEYS}
|
||||
if not filtered_curator:
|
||||
return None
|
||||
return {"curator": filtered_curator}
|
||||
|
||||
|
||||
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
|
||||
|
||||
@@ -86,6 +86,75 @@ class PrivateSquareSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
#: Id of the always-present built-in default square (used when the admin has
|
||||
#: configured no squares, or when a published item picks none).
|
||||
DEFAULT_SQUARE_ID = "default"
|
||||
|
||||
|
||||
class PublishSquare(BaseModel):
|
||||
"""One admin-configured publish square (发布广场).
|
||||
|
||||
A pure organizational category that published agents/skills can be filed
|
||||
under — unlike the password-gated :class:`PrivateSquareSettings`, a square
|
||||
has **no access control**: every logged-in user can browse every square.
|
||||
Each agent/skill belongs to at most one square (a single ``square_id``
|
||||
field on the row), so an item can never be "published twice".
|
||||
"""
|
||||
|
||||
id: str = Field(..., description="Stable square id (slug/uuid); used as agents.square_id / skills.square_id")
|
||||
name: str = Field(..., description="Display name shown as the square's tab label")
|
||||
description: str = Field(default="", description="Optional description shown in the admin config / tab hint")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class PublishSquaresSettings(BaseModel):
|
||||
"""Admin-configurable list of publish squares.
|
||||
|
||||
``squares`` is display-ordered. ``default_square_id`` selects the square a
|
||||
publisher lands in when they do not pick one explicitly. There is always at
|
||||
least one effective square — :func:`list_publish_squares` injects a built-in
|
||||
"默认广场" when the admin has configured none, so the publish flow always has
|
||||
a valid target.
|
||||
"""
|
||||
|
||||
squares: list[PublishSquare] = Field(default_factory=list, description="Ordered list of configured squares")
|
||||
default_square_id: str = Field(default=DEFAULT_SQUARE_ID, description="Square id used when a publisher picks none")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
def list_publish_squares(settings: PublishSquaresSettings) -> list[PublishSquare]:
|
||||
"""Return the effective square list, always non-empty.
|
||||
|
||||
Falls back to a single built-in default square when the admin has not
|
||||
configured any, so callers never have to special-case the empty state.
|
||||
"""
|
||||
if settings.squares:
|
||||
return list(settings.squares)
|
||||
return [PublishSquare(id=DEFAULT_SQUARE_ID, name="默认广场", description="")]
|
||||
|
||||
|
||||
def resolve_default_square_id(settings: PublishSquaresSettings) -> str:
|
||||
"""Resolve the effective default square id against the configured set."""
|
||||
ids = [s.id for s in settings.squares]
|
||||
if settings.default_square_id and settings.default_square_id in ids:
|
||||
return settings.default_square_id
|
||||
return ids[0] if ids else DEFAULT_SQUARE_ID
|
||||
|
||||
|
||||
def normalize_square_id(square_id: str | None, settings: PublishSquaresSettings) -> str:
|
||||
"""Coerce an arbitrary ``square_id`` to a valid configured square id.
|
||||
|
||||
Empty / unknown / orphaned (square later deleted) ids all collapse to the
|
||||
effective default square, so a published item always resolves to a real
|
||||
tab.
|
||||
"""
|
||||
default_id = resolve_default_square_id(settings)
|
||||
if not square_id:
|
||||
return default_id
|
||||
valid = {s.id for s in list_publish_squares(settings)}
|
||||
return square_id if square_id in valid else default_id
|
||||
|
||||
|
||||
class LeaderboardSettings(BaseModel):
|
||||
"""Admin-configurable defaults for the user leaderboard analytics."""
|
||||
|
||||
@@ -102,6 +171,7 @@ class SystemSettings(BaseModel):
|
||||
|
||||
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
|
||||
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
|
||||
publish_squares: PublishSquaresSettings = Field(default_factory=PublishSquaresSettings)
|
||||
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
|
||||
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
|
||||
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
|
||||
|
||||
@@ -135,7 +135,15 @@ def to_cypher(graph: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def to_html(graph: dict[str, Any]) -> str:
|
||||
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
|
||||
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
|
||||
|
||||
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
|
||||
can stay in sync with the app's day/night mode (an iframe cannot inherit the
|
||||
parent's CSS). The layout settles and then **freezes** so a dense graph stops
|
||||
drifting and stays clickable; node size scales with degree; and hovering or
|
||||
clicking a node focuses it + its direct neighbours while dimming everything
|
||||
else, which cuts through heavy node/edge clutter.
|
||||
"""
|
||||
data_json = json.dumps(graph, ensure_ascii=False)
|
||||
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
|
||||
|
||||
@@ -178,50 +186,250 @@ _HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Knowledge Graph</title>
|
||||
<style>
|
||||
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
|
||||
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
|
||||
canvas{display:block}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
|
||||
:root{
|
||||
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
|
||||
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
|
||||
}
|
||||
html.light{
|
||||
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
|
||||
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
|
||||
}
|
||||
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
canvas{display:block;cursor:grab}
|
||||
canvas.grabbing{cursor:grabbing}
|
||||
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
|
||||
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
|
||||
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
|
||||
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
|
||||
#legend .dot{width:10px;height:10px;border-radius:50%}
|
||||
#tip{padding:4px 8px;pointer-events:none;display:none}
|
||||
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
|
||||
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
|
||||
#panel .meta{color:var(--sub);margin-bottom:8px}
|
||||
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
|
||||
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
|
||||
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
|
||||
#reset:hover{filter:brightness(1.1)}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="hud"></div>
|
||||
<div id="tip"></div>
|
||||
<div id="hud" class="card"></div>
|
||||
<div id="legend" class="card">
|
||||
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
|
||||
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
|
||||
</div>
|
||||
<div id="tip" class="card"></div>
|
||||
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
|
||||
<div id="reset" class="card">重置视图</div>
|
||||
<canvas id="c"></canvas>
|
||||
<script>
|
||||
const GRAPH = __GRAPH_DATA__;
|
||||
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
|
||||
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
|
||||
const PAL = {
|
||||
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
|
||||
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
|
||||
}[THEME];
|
||||
document.documentElement.classList.toggle('light', THEME === 'light');
|
||||
document.getElementById('lg-note').style.background = PAL.note;
|
||||
document.getElementById('lg-entity').style.background = PAL.entity;
|
||||
|
||||
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
|
||||
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
|
||||
let W=0,H=0;
|
||||
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
|
||||
addEventListener('resize',resize);resize();
|
||||
const panel = document.getElementById('panel');
|
||||
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
|
||||
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
|
||||
addEventListener('resize',()=>{resize();bump();});resize();
|
||||
|
||||
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
|
||||
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
|
||||
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
|
||||
let cam={x:0,y:0,k:1},drag=null,pan=null;
|
||||
function step(){
|
||||
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
|
||||
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
|
||||
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
|
||||
|
||||
// --- index, adjacency, degree, node sizing -------------------------------
|
||||
const idx={}, adj={};
|
||||
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
|
||||
const edges=[];
|
||||
for(const e of GRAPH.edges){
|
||||
const s=idx[e.source], t=idx[e.target];
|
||||
if(!s||!t||s===t) continue;
|
||||
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
|
||||
adj[e.source].add(e.target); adj[e.target].add(e.source);
|
||||
}
|
||||
function draw(){
|
||||
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
|
||||
ctx.strokeStyle='rgba(120,150,255,.25)';
|
||||
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
|
||||
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
|
||||
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
|
||||
const N=GRAPH.nodes.length;
|
||||
GRAPH.nodes.forEach((n,i)=>{
|
||||
n.deg=adj[n.id].size;
|
||||
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
|
||||
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
|
||||
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
|
||||
});
|
||||
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
|
||||
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
|
||||
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
|
||||
|
||||
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
|
||||
const REP=9000, LINK=92, CENTER=0.0016;
|
||||
let alpha=1, drag=null;
|
||||
function physics(){
|
||||
for(let i=0;i<N;i++){
|
||||
const n=GRAPH.nodes[i];
|
||||
for(let j=i+1;j<N;j++){
|
||||
const m=GRAPH.nodes[j];
|
||||
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
|
||||
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
|
||||
}
|
||||
}
|
||||
for(const e of edges){
|
||||
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
|
||||
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
|
||||
}
|
||||
for(const n of GRAPH.nodes){
|
||||
if(n===drag){n.vx=n.vy=0;continue;}
|
||||
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
|
||||
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
|
||||
}
|
||||
alpha*=0.985;
|
||||
}
|
||||
function loop(){step();draw();requestAnimationFrame(loop);}loop();
|
||||
|
||||
// --- camera / fit-to-view -------------------------------------------------
|
||||
let cam={x:0,y:0,k:1};
|
||||
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
|
||||
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
|
||||
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
|
||||
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
|
||||
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
|
||||
addEventListener('mouseup',()=>{drag=null;pan=null;});
|
||||
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
|
||||
function fitView(){
|
||||
if(!N){return;}
|
||||
let a=1e9,b=1e9,c=-1e9,d=-1e9;
|
||||
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
|
||||
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
|
||||
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
|
||||
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
|
||||
}
|
||||
let fitted=false;
|
||||
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
|
||||
|
||||
// --- focus / hover state --------------------------------------------------
|
||||
let hoverNode=null, focusNode=null;
|
||||
function active(){return focusNode||hoverNode;}
|
||||
function neighbors(n){return n?adj[n.id]:null;}
|
||||
|
||||
function draw(){
|
||||
ctx.setTransform(DPR,0,0,DPR,0,0);
|
||||
ctx.clearRect(0,0,W,H);
|
||||
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
|
||||
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
|
||||
const [er,eg,eb]=PAL.edge;
|
||||
|
||||
// edges: clearly visible by default; when a node is focused, only its edges
|
||||
// stay bright and the rest fade back (but never to invisible).
|
||||
for(const e of edges){
|
||||
const lit = act && (e.s===act||e.t===act);
|
||||
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
|
||||
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
|
||||
// never thin out to a hairline when zoomed in or on dense graphs.
|
||||
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
|
||||
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
|
||||
ctx.lineWidth=Math.max(1,screenW)/cam.k;
|
||||
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
|
||||
}
|
||||
// nodes
|
||||
for(const n of GRAPH.nodes){
|
||||
const dim = act && n!==act && !(nb&&nb.has(n.id));
|
||||
ctx.globalAlpha = dim?0.22:1;
|
||||
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
|
||||
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
|
||||
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
|
||||
ctx.globalAlpha=1;
|
||||
}
|
||||
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
|
||||
ctx.textBaseline='middle';
|
||||
for(const n of GRAPH.nodes){
|
||||
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
|
||||
if(!show) continue;
|
||||
const txt=(n.label||'').slice(0,20);
|
||||
if(!txt) continue;
|
||||
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
|
||||
const tx=n.x+n.r+4/cam.k, ty=n.y;
|
||||
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
|
||||
ctx.strokeText(txt,tx,ty);
|
||||
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
|
||||
}
|
||||
}
|
||||
|
||||
function loop(){
|
||||
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
|
||||
draw();
|
||||
requestAnimationFrame(loop);
|
||||
}
|
||||
// Pre-settle a few iterations so the first paint is already roughly laid out.
|
||||
for(let i=0;i<60;i++)physics();
|
||||
fitView();
|
||||
loop();
|
||||
|
||||
// --- picking & interaction ------------------------------------------------
|
||||
function pick(p){
|
||||
let best=null,bd=1e9;
|
||||
for(const n of GRAPH.nodes){
|
||||
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
|
||||
if(d<hit*hit && d<bd){bd=d;best=n;}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
let pan=null, downXY=null, moved=false;
|
||||
canvas.addEventListener('mousedown',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
|
||||
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
|
||||
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
|
||||
canvas.classList.add('grabbing');
|
||||
});
|
||||
addEventListener('mousemove',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY);
|
||||
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
|
||||
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
|
||||
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
hoverNode=drag||pan?hoverNode:pick(p);
|
||||
if(hoverNode && !drag && !pan){
|
||||
tip.style.display='block';
|
||||
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
|
||||
tip.style.top=(ev.clientY+14)+'px';
|
||||
tip.textContent=hoverNode.label||'';
|
||||
} else { tip.style.display='none'; }
|
||||
});
|
||||
addEventListener('mouseup',ev=>{
|
||||
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
|
||||
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
|
||||
renderPanel();
|
||||
}
|
||||
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
|
||||
});
|
||||
canvas.addEventListener('wheel',ev=>{
|
||||
ev.preventDefault();
|
||||
const s=ev.deltaY<0?1.12:0.89;
|
||||
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
|
||||
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
|
||||
},{passive:false});
|
||||
|
||||
// --- detail panel ---------------------------------------------------------
|
||||
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
|
||||
function renderPanel(){
|
||||
if(!focusNode){panel.style.display='none';return;}
|
||||
const n=focusNode;
|
||||
panel.querySelector('h3').textContent=n.label||n.id;
|
||||
const cat=n.category?' · '+n.category:'';
|
||||
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
|
||||
const nb=panel.querySelector('.nb'); nb.innerHTML='';
|
||||
let count=0;
|
||||
for(const id of adj[n.id]){
|
||||
if(count++>=24)break;
|
||||
const m=idx[id]; if(!m)continue;
|
||||
const s=document.createElement('span'); s.textContent=m.label||id;
|
||||
s.title=m.label||id;
|
||||
s.style.cursor='pointer';
|
||||
s.onclick=()=>{focusNode=m;renderPanel();};
|
||||
nb.appendChild(s);
|
||||
}
|
||||
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
|
||||
panel.style.display='block';
|
||||
}
|
||||
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
|
||||
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
+146
-13
@@ -84,6 +84,46 @@ references(外部来源汇总)、decision(架构/设计决策)、journal
|
||||
}"""
|
||||
|
||||
|
||||
_DOC_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是照抄原文。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
|
||||
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
|
||||
- 用自己的话重组为结构化知识,不要逐段复制原文。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
|
||||
- 文档没有可沉淀的知识时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "文档背景或适用场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
@@ -171,21 +211,27 @@ def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _build_note_draft(
|
||||
note_data: dict[str, Any],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None,
|
||||
turns: list[dict[str, str]],
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
source_key: str,
|
||||
fallback_title: str,
|
||||
title_override: str | None,
|
||||
tags: list[str] | None,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
source_key_suffix: str = "",
|
||||
) -> NoteDraft:
|
||||
"""Render one note dict into a :class:`NoteDraft`."""
|
||||
"""Render one note dict into a :class:`NoteDraft`.
|
||||
|
||||
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
|
||||
``source_key`` and a ``fallback_title`` (used when neither the user override
|
||||
nor the LLM produced a usable title), so both thread and document capture
|
||||
share one renderer.
|
||||
"""
|
||||
category = str(note_data.get("category") or "synthesis").strip().lower()
|
||||
if category not in _VALID_CATEGORIES:
|
||||
category = "synthesis"
|
||||
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
|
||||
summary = str(note_data.get("summary") or "").strip()
|
||||
related = extractor.related_links(project_name)
|
||||
body = _render_body(note_data, final_title, related, sources)
|
||||
@@ -210,15 +256,15 @@ def _build_note_draft(
|
||||
summary=summary or final_title,
|
||||
content_md=body,
|
||||
category=category,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
tags=base_tags,
|
||||
confidence=max(0.0, min(confidence, 1.0)),
|
||||
related=related,
|
||||
sources=sources,
|
||||
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
|
||||
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
|
||||
source_key=f"thread:{thread_id}{source_key_suffix}",
|
||||
source_key=source_key,
|
||||
)
|
||||
|
||||
|
||||
@@ -293,6 +339,7 @@ async def extract_thread_llm_multi(
|
||||
else:
|
||||
note_dicts = note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = extractor._auto_title(thread_title, turns)
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
# The user-supplied title override only applies to a single-note result;
|
||||
@@ -302,14 +349,100 @@ async def extract_thread_llm_multi(
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
turns=turns,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_key=f"thread:{thread_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_document_llm_multi(
|
||||
text: str,
|
||||
*,
|
||||
doc_id: str,
|
||||
doc_title: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill an imported **document** into one *or several* knowledge notes.
|
||||
|
||||
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
|
||||
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
|
||||
when the LLM path fails (callers fall back to a verbatim note); raises
|
||||
:class:`LLMRejectedIngest` when the model deliberately judges the document
|
||||
worthless. Resulting notes carry ``source_type="file"``.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
|
||||
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
|
||||
prompt = [
|
||||
SystemMessage(content=_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
|
||||
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
doc_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
source_key=f"file:{doc_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
source_key_suffix=suffix,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
@@ -394,6 +394,129 @@ class KnowledgeService:
|
||||
)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft:
|
||||
"""Build a verbatim fallback note from a file when LLM distillation fails.
|
||||
|
||||
Lands as a ``draft`` (like the thread fallback) since it is the raw
|
||||
document, not distilled knowledge.
|
||||
"""
|
||||
base_tags = list(tags or [])
|
||||
for default in (self.project_name, "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
name = (title or Path(filename).stem or "文档知识").strip() or "文档知识"
|
||||
summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280]
|
||||
return NoteDraft(
|
||||
title=name,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="references",
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
status="draft",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=extractor.related_links(self.project_name),
|
||||
source_key=f"file:{doc_id}",
|
||||
)
|
||||
|
||||
async def capture_file(
|
||||
self,
|
||||
*,
|
||||
content_md: str,
|
||||
filename: str,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
created_by: str | None = None,
|
||||
model_name: str | None = None,
|
||||
skip_existing: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Import a file (already converted to Markdown/text) into knowledge notes.
|
||||
|
||||
Runs the same wiki-capture LLM distillation as thread capture, so an
|
||||
uploaded document becomes one or more declarative wiki notes in the vault.
|
||||
Falls back to a verbatim ``draft`` note when the LLM is disabled or returns
|
||||
nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``;
|
||||
raises :class:`EmptyThreadError` when there is nothing to keep.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
text = (content_md or "").strip()
|
||||
if not text:
|
||||
raise EmptyThreadError(f"File {filename} is empty")
|
||||
# Strip any frontmatter the document carries; the vault writer adds its own.
|
||||
_, parsed = MarkdownVault.parse(text)
|
||||
body = (parsed or text).strip()
|
||||
if not body:
|
||||
raise EmptyThreadError(f"File {filename} has no body content")
|
||||
# Stable id from name + size so re-importing the same file can be skipped.
|
||||
doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
logger.info(
|
||||
"[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)",
|
||||
filename,
|
||||
len(body),
|
||||
doc_id,
|
||||
self.config.llm_extract_enabled,
|
||||
skip_existing,
|
||||
)
|
||||
|
||||
if skip_existing:
|
||||
existing = await self.repo.find_by_source("file", doc_id)
|
||||
if existing:
|
||||
logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id"))
|
||||
return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False}
|
||||
|
||||
drafts: list[NoteDraft] = []
|
||||
llm_fallback = False
|
||||
if self.config.llm_extract_enabled:
|
||||
try:
|
||||
drafts = await llm_extractor.extract_document_llm_multi(
|
||||
body,
|
||||
doc_id=doc_id,
|
||||
doc_title=title or Path(filename).stem,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
model_name=model_name or self.config.extract_model_name,
|
||||
max_input_chars=self.config.extract_max_input_chars,
|
||||
allow_multi=self.config.multi_note_enabled,
|
||||
)
|
||||
except llm_extractor.LLMRejectedIngest as exc:
|
||||
raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc
|
||||
if not drafts:
|
||||
llm_fallback = True
|
||||
if not drafts:
|
||||
drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)]
|
||||
|
||||
effective_status = status
|
||||
if llm_fallback and normalize_status(status) == "approved":
|
||||
effective_status = "draft"
|
||||
normalized = normalize_status(effective_status)
|
||||
for draft in drafts:
|
||||
draft.status = normalized
|
||||
|
||||
# Cross-link sibling notes split from the same document (resolved by title
|
||||
# once all are persisted), mirroring multi-note thread capture.
|
||||
if len(drafts) > 1:
|
||||
for i, draft in enumerate(drafts):
|
||||
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
|
||||
if siblings:
|
||||
draft.content_md = _inject_related_notes(draft.content_md, siblings)
|
||||
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
|
||||
|
||||
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
|
||||
logger.info(
|
||||
"[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s",
|
||||
filename,
|
||||
len(notes),
|
||||
", ".join(n.get("title", "") for n in notes),
|
||||
normalized,
|
||||
llm_fallback,
|
||||
)
|
||||
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
|
||||
|
||||
# -- CRUD -------------------------------------------------------------
|
||||
async def list_notes(
|
||||
self,
|
||||
|
||||
@@ -131,6 +131,8 @@ class AgentStore(ABC):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
|
||||
+16
-3
@@ -61,6 +61,8 @@ class MemoryAgentStore(AgentStore):
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
@@ -93,7 +95,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -103,7 +105,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -159,7 +161,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -198,16 +200,27 @@ class MemoryAgentStore(AgentStore):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
|
||||
+23
-17
@@ -27,6 +27,12 @@ class AgentRow(Base):
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Publish square (发布广场) this agent is filed under. A pure organizational
|
||||
# category with no access control — independent of ``published`` /
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
# it as the admin-configured default square. Each agent belongs to at most
|
||||
# one square (single column), so it can never be published to two squares.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
@@ -37,7 +43,7 @@ class AgentRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentFavoriteRow(Base):
|
||||
class AgentFavoriteRow(Base):
|
||||
"""A user's bookmark of a (typically public-square) agent.
|
||||
|
||||
Lets a user collect agents from the 广场 (square) into their "我的" tab
|
||||
@@ -51,22 +57,22 @@ class AgentFavoriteRow(Base):
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentPinRow(Base):
|
||||
"""A per-user pin that keeps an agent near the top of that user's lists."""
|
||||
|
||||
__tablename__ = "agent_pins"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentSkillRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentPinRow(Base):
|
||||
"""A per-user pin that keeps an agent near the top of that user's lists."""
|
||||
|
||||
__tablename__ = "agent_pins"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentSkillRow(Base):
|
||||
"""Denormalized cache of a custom agent's selected skills.
|
||||
|
||||
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
|
||||
|
||||
+14
-3
@@ -93,6 +93,7 @@ class AgentRepository(AgentStore):
|
||||
user_id=data.get("user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
is_private=bool(data.get("is_private", False)),
|
||||
square_id=str(data.get("square_id") or ""),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -131,7 +132,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -144,7 +145,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -224,7 +225,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -290,6 +291,8 @@ class AgentRepository(AgentStore):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
filters = []
|
||||
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
|
||||
@@ -302,6 +305,14 @@ class AgentRepository(AgentStore):
|
||||
if owner and owner.strip():
|
||||
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
|
||||
filters.append(AgentRow.user_id == owner.strip())
|
||||
if square_id and square_id.strip():
|
||||
sid = square_id.strip()
|
||||
if square_is_default:
|
||||
# Legacy / unassigned rows (square_id == "") belong to the
|
||||
# default square, so include them alongside explicit matches.
|
||||
filters.append(or_(AgentRow.square_id == sid, AgentRow.square_id == ""))
|
||||
else:
|
||||
filters.append(AgentRow.square_id == sid)
|
||||
|
||||
count_stmt = select(func.count()).select_from(AgentRow)
|
||||
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Add ``agents.square_id`` and ``skills.square_id`` for the publish-square (发布广场) feature.
|
||||
|
||||
A nullable-default (server_default '') string column tagging which admin-
|
||||
configured publish square an agent / skill is filed under. Pure organizational
|
||||
category — independent of the ``published`` / ``is_private`` flags. Empty string
|
||||
means "unassigned" and is resolved to the default square at read time.
|
||||
|
||||
Revision ID: 20260611_01
|
||||
Revises: 20260609_02
|
||||
Create Date: 2026-06-11
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260611_01"
|
||||
down_revision = "20260609_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def _add_square_id(table_name: str, index_name: str) -> None:
|
||||
if _has_table(table_name) and not _has_column(table_name, "square_id"):
|
||||
op.add_column(
|
||||
table_name,
|
||||
sa.Column(
|
||||
"square_id",
|
||||
sa.String(length=64),
|
||||
nullable=False,
|
||||
server_default="",
|
||||
),
|
||||
)
|
||||
if _has_table(table_name) and not _has_index(table_name, index_name):
|
||||
op.create_index(index_name, table_name, ["square_id"])
|
||||
|
||||
|
||||
def _drop_square_id(table_name: str, index_name: str) -> None:
|
||||
if _has_index(table_name, index_name):
|
||||
op.drop_index(index_name, table_name=table_name)
|
||||
if _has_column(table_name, "square_id"):
|
||||
op.drop_column(table_name, "square_id")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_add_square_id("agents", "ix_agents_square_id")
|
||||
_add_square_id("skills", "ix_skills_square_id")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
_drop_square_id("skills", "ix_skills_square_id")
|
||||
_drop_square_id("agents", "ix_agents_square_id")
|
||||
+5
@@ -71,6 +71,7 @@ class MemorySkillStore(SkillStore):
|
||||
"published": bool(data.get("published", False)),
|
||||
"detail": data.get("detail"),
|
||||
"name_zh": data.get("name_zh"),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -106,6 +107,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
record["square_id"] = str(data["square_id"] or "")
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
@@ -121,6 +124,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
record["square_id"] = str(data["square_id"] or "")
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@@ -33,6 +33,10 @@ class SkillRow(Base):
|
||||
# the on-disk SKILL.md) so it can be set for built-in skills too, and so the
|
||||
# shared source file is never rewritten.
|
||||
name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
|
||||
# Publish square (发布广场) this skill is filed under once published. Pure
|
||||
# organizational category, no access control. Empty string = unassigned →
|
||||
# the listing layer resolves it to the default square. One square per skill.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ class SkillRepository(SkillStore):
|
||||
published=bool(data.get("published", False)),
|
||||
detail=data.get("detail"),
|
||||
name_zh=data.get("name_zh"),
|
||||
square_id=str(data.get("square_id") or ""),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -109,6 +110,8 @@ class SkillRepository(SkillStore):
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
row.square_id = str(data["square_id"] or "")
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
@@ -127,6 +130,8 @@ class SkillRepository(SkillStore):
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
row.square_id = str(data["square_id"] or "")
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Post-turn background skill review worker.
|
||||
|
||||
After a conversation turn completes, SkillReviewMiddleware calls
|
||||
``maybe_start_review()``. This module runs a lightweight LLM React agent
|
||||
(with skill_manage + skill_list tools) that reads the conversation history
|
||||
and decides whether to create or patch skills.
|
||||
|
||||
Concurrency model: per-thread asyncio.Lock + a single _pending slot.
|
||||
If a review is already running for the thread, the newer messages overwrite
|
||||
the pending slot; once the current review finishes, it drains that slot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import textwrap
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from langchain_core.messages import BaseMessage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_locks: dict[str, asyncio.Lock] = {}
|
||||
_pending: dict[str, list[BaseMessage]] = {}
|
||||
|
||||
|
||||
def _get_lock(thread_id: str) -> asyncio.Lock:
|
||||
if thread_id not in _locks:
|
||||
_locks[thread_id] = asyncio.Lock()
|
||||
return _locks[thread_id]
|
||||
|
||||
|
||||
def extract_tool_call_count(messages: list[BaseMessage]) -> int:
|
||||
"""Count total tool calls across all AI messages in the conversation."""
|
||||
count = 0
|
||||
for msg in messages:
|
||||
tool_calls: Any = None
|
||||
if hasattr(msg, "tool_calls"):
|
||||
tool_calls = msg.tool_calls
|
||||
elif isinstance(msg, dict):
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls:
|
||||
count += len(tool_calls)
|
||||
return count
|
||||
|
||||
|
||||
def _format_messages_for_review(messages: list[BaseMessage], max_messages: int) -> str:
|
||||
"""Serialize recent messages to a readable text block for the LLM reviewer."""
|
||||
recent = messages[-max_messages:]
|
||||
lines: list[str] = []
|
||||
for msg in recent:
|
||||
role = type(msg).__name__.replace("Message", "")
|
||||
content = msg.content if isinstance(msg.content, str) else str(msg.content)
|
||||
tool_calls = getattr(msg, "tool_calls", None)
|
||||
if tool_calls:
|
||||
calls_str = ", ".join(
|
||||
f"{tc.name if hasattr(tc, 'name') else tc.get('name', '?')}()"
|
||||
for tc in tool_calls
|
||||
)
|
||||
lines.append(f"[{role}] <tool_calls: {calls_str}>")
|
||||
else:
|
||||
preview = content[:400].replace("\n", " ")
|
||||
lines.append(f"[{role}] {preview}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
_REVIEW_SYSTEM_PROMPT = textwrap.dedent("""\
|
||||
# Role: 技能评审代理(保守模式)
|
||||
|
||||
## Profile
|
||||
- language: zh-CN
|
||||
- description: 审阅已完成对话,仅在出现强证据的高价值可复用工作流时才沉淀为技能。**默认决策是 do-nothing** —— 宁可漏掉一个普通技能,也绝不创建垃圾技能。
|
||||
- background: 历史问题:之前版本触发过于宽松,沉淀了大量低价值技能,污染了技能库。本版本针对性收紧。
|
||||
- personality: 极度保守、苛刻挑剔、宁缺勿滥;面对模糊场景默认 do-nothing。
|
||||
- expertise: 高价值工作流识别、严格去重、SKILL.md 规范、do-nothing 判定。
|
||||
- target_audience: 技能库维护者、平台工程师。
|
||||
|
||||
## Skills
|
||||
|
||||
1. 工作流抽象与评审
|
||||
- 价值评估: 区分一次性数据请求与具有沉淀价值的工作流。
|
||||
- 去数据化归纳: 抽象出与具体数据无关的通用步骤、参数与注意事项。
|
||||
- 决策映射: 在 do-nothing / patch / create 三者中做出可解释的唯一决定。
|
||||
|
||||
2. 文档与规范控制
|
||||
- SKILL.md 编写: 简洁步骤 + 命令 + 坑点提示(非对话稿)。
|
||||
- 命名规范: kebab-case(小写字母/数字/连字符,≤64 字符)。
|
||||
- 触发描述: 以 "When" 开头的单句(≤1024 字符、无尖括号)。
|
||||
- 结构校验: frontmatter 允许键、name 一致性、首尾分隔线。
|
||||
|
||||
3. 工具使用与错误处理
|
||||
- skill_list 优先调用,识别功能重叠与现有覆盖范围。
|
||||
- skill_manage 正确选 action=create / patch,并传递合规内容。
|
||||
- 报错诊断与修正后才能重试,禁止内容不变重提。
|
||||
|
||||
4. 质量控制与安全
|
||||
- 去重策略: 少而精,避免噪声与重复。
|
||||
- 隐私合规: 严禁嵌入用户特定数据、文件路径、凭据。
|
||||
- 一致性: name / description / content 三者语义对齐。
|
||||
|
||||
## Rules
|
||||
|
||||
### 0. 默认决策
|
||||
|
||||
**默认就是 do-nothing。** 仅当下面"必要条件"全部满足时才考虑创建;任一不满足 → do-nothing。垃圾技能比少一个技能更糟糕。
|
||||
|
||||
### 1. 创建必要条件(必须全部满足,AND 关系)
|
||||
|
||||
要创建一个新技能,以下条件**必须同时**满足:
|
||||
|
||||
1. **多步且有依赖**: 对话中至少 5 次工具调用,且步骤间存在依赖关系(不是并行查询,也不是同一工具的简单循环)。
|
||||
2. **包含纠错经验**: 包含至少一次失败 / 报错 / 意外结果,agent 通过修正后达到目标。这种"踩坑—纠错"才是有沉淀价值的知识。
|
||||
3. **跨场景可复用**: 抽离掉对话中的具体数据后,步骤本身在不同输入 / 不同数据集下仍然成立。
|
||||
4. **不与现有技能重叠**: skill_list 中没有功能覆盖率 ≥70% 的现有技能(若有,应当 patch 而非新建)。
|
||||
5. **能写出有料的 SKILL.md**: 抽象后能产出 ≥300 字、包含具体步骤 / 命令 / 坑点的工作流指南。写不出说明本来就没料。
|
||||
|
||||
### 2. 明确不要创建(即使工具调用很多也不要建)
|
||||
|
||||
- **单工具循环**: 连续多次 web_search 都只是搜索,没有形成工作流。
|
||||
- **一次性数据查询**: 仅是"搜索 + 总结"模式,没有可复用结构。
|
||||
- **LLM 常识可答**: 如"如何配置 Python 虚拟环境"这类公开文档 / 通识问题。
|
||||
- **流程过度通用**: 如"先搜索再总结再回答"这种描述等于没说。
|
||||
- **场景过窄**: 只在某一特定数据集 / 系统 / 项目下有效,换环境就失效。
|
||||
- **内容字数不足**: 写不出 ≥300 字含真干货的 SKILL.md。
|
||||
- **仅为"做点什么"而建**: 找不到清晰理由就 do-nothing,不要为了"完成 review"而强行创建。
|
||||
|
||||
### 3. 创建前自检(4 题,全部"是"才能 create)
|
||||
|
||||
提交 create 前在心中依次回答以下问题,任一为"否" → 立即放弃创建、改 do-nothing:
|
||||
|
||||
1. 如果我不创建这个技能,下一个遇到类似任务的用户会损失什么?(答得上来才证明有沉淀价值)
|
||||
2. 这个技能是否能被一句话简短复述?若"能",说明信息量不足 → do-nothing。
|
||||
3. SKILL.md 写完后是否包含至少 3 个具体步骤或命令?
|
||||
4. 步骤中是否含有非显而易见的坑点提示(不是 LLM 凭常识就能猜到的)?
|
||||
|
||||
### 4. 单次 review 上限
|
||||
|
||||
**每次 review 最多创建 1 条新技能。** 即便发现多个候选,也只挑最有价值的那一个,其余 do-nothing。这是硬约束,不接受例外。
|
||||
|
||||
### 5. Patch 优先于 Create
|
||||
|
||||
若 skill_list 中已存在功能相近(即便不完全相同)的技能,优先选择 patch 现有技能补充改进点,而非新建。新技能名额只留给**全新类别**的工作流。
|
||||
|
||||
### 6. 格式与命名硬约束
|
||||
|
||||
- 命名:kebab-case,仅含小写字母 / 数字 / 连字符,≤64 字符(如 git-bisect-bug-hunt、api-load-test)。
|
||||
- description:以 "When " 开头,单句,≤1024 字符,禁止尖括号 < >。
|
||||
- content:合法 SKILL.md,包含 frontmatter 与正文。
|
||||
- frontmatter 仅允许键:name、description、license、allowed-tools、metadata、compatibility、version、author。
|
||||
- frontmatter 首行与末行必须为 "---" 独占一行。
|
||||
- frontmatter 的 name 字段必须与 skill_manage 调用的 name 参数**完全一致**;不一致直接调用失败。
|
||||
- 禁止添加 tags、category、type 等额外键。
|
||||
- content 必须是简洁工作流指南,禁止对话转录。
|
||||
- 严禁包含用户特定数据、文件路径、凭据。
|
||||
|
||||
### 7. 错误处理
|
||||
|
||||
- 调用 skill_manage 报错 → 阅读错误信息,定位根因,**修正内容后**再重试。
|
||||
- 同名已存在 → 改用 action=patch(直接 create 同名会失败)。
|
||||
- 禁止在未修改内容的情况下重复提交。
|
||||
|
||||
## Workflows
|
||||
|
||||
### 目标
|
||||
仅在发现**高价值、可复用、有踩坑沉淀**的工作流时,创建或修补一条技能;其余一律 do-nothing。
|
||||
|
||||
### 步骤 1: 获取基线
|
||||
调用 skill_list,记录现有技能名称与功能覆盖范围。这一步在每次 review 中必须执行。
|
||||
|
||||
### 步骤 2: 价值初筛(三关,任一不过即 do-nothing 结束)
|
||||
通读对话并依次确认:
|
||||
- 工具调用总数是否 ≥5?否则 do-nothing 结束。
|
||||
- 是否包含失败 / 报错 / 修正过程?否则 do-nothing 结束。
|
||||
- 步骤间是否存在依赖关系?否则 do-nothing 结束。
|
||||
|
||||
只有三关全过才进入步骤 3。
|
||||
|
||||
### 步骤 3: 候选抽象
|
||||
将对话抽象为候选技能要素:
|
||||
- 候选名称(kebab-case)
|
||||
- 触发描述(When ... )
|
||||
- 关键步骤、命令、坑点提示
|
||||
- 剔除所有数据特定内容与隐私信息
|
||||
|
||||
### 步骤 4: 重叠检查
|
||||
与 skill_list 比对:
|
||||
- 完全无重叠 → 候选 action=create,进入步骤 5。
|
||||
- 部分重叠(功能覆盖 ≥70%)→ 候选 action=patch,跳到步骤 6。
|
||||
- 完全覆盖 → do-nothing 结束。
|
||||
|
||||
### 步骤 5: 创建前自检
|
||||
若候选为 create,依次回答 Rules 第 3 节的 4 个自检问题。任一为"否" → 立即放弃创建,do-nothing 结束。全部为"是"才进入步骤 6。
|
||||
|
||||
### 步骤 6: 提交
|
||||
- create:调用 skill_manage(action="create", name, content)。
|
||||
- patch:调用 skill_manage(action="patch", name, content),仅针对差异 / 改进点。
|
||||
- 一次 review 最多一次写操作。
|
||||
|
||||
### 步骤 7: 错误处理
|
||||
若报错,按 Rules 第 7 节处理:阅读错误信息 → 修正根因 → 再试。禁止内容不变重提。
|
||||
|
||||
### 预期结果
|
||||
- **多数情况下**: do-nothing。这是好的结果,不是失败。
|
||||
- **少数情况下**: 一条高价值新技能,或对现有技能的一次有针对性 patch。
|
||||
- **永远不要**: 为了"做点什么"而创建低价值技能。
|
||||
|
||||
## Initialization
|
||||
作为技能评审代理(保守模式),你必须严格遵守 Rules,按 Workflows 执行。**默认决策是 do-nothing。** 只有强证据完全满足所有创建条件时,才考虑创建。创建一条垃圾技能比放弃一条普通技能更糟糕。
|
||||
""")
|
||||
|
||||
|
||||
async def _run_review(thread_id: str, messages: list[BaseMessage], app_config: "AppConfig", user_id: str = "default") -> None:
|
||||
"""Core review logic: create_agent with full middleware chain, no 'silent creation' constraint.
|
||||
|
||||
Uses create_agent (the deerflow agent factory) directly so we can:
|
||||
- Reuse the full middleware chain (ToolErrorHandlingMiddleware, etc.)
|
||||
- Reuse the full tool set
|
||||
- But pass app_config with skill_evolution.enabled=False to strip the
|
||||
"Never create skills silently without user confirmation" constraint
|
||||
from the system prompt.
|
||||
Review instructions are injected via PromptPrefixMiddleware through
|
||||
additional_kwargs["prompt_prefix"] on the HumanMessage.
|
||||
"""
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.tools.tools import get_available_tools
|
||||
|
||||
evolution_cfg = app_config.skill_evolution
|
||||
timeout = evolution_cfg.review_worker.timeout_seconds
|
||||
max_messages = evolution_cfg.review_worker.max_messages
|
||||
model_name = evolution_cfg.review_worker.model_name
|
||||
|
||||
conversation_text = _format_messages_for_review(messages, max_messages)
|
||||
|
||||
# Disable skill_evolution so the system prompt does NOT include
|
||||
# "Never create skills silently without user confirmation".
|
||||
# Also prevents SkillReviewMiddleware from registering (no recursive reviews).
|
||||
review_app_config = app_config.model_copy(deep=True)
|
||||
review_app_config.skill_evolution.enabled = False
|
||||
|
||||
try:
|
||||
model = create_chat_model(name=model_name, app_config=review_app_config)
|
||||
except Exception:
|
||||
logger.warning("skill_review_worker: could not load model %r, aborting review", model_name)
|
||||
return
|
||||
|
||||
tools = get_available_tools(app_config=review_app_config, subagent_enabled=False)
|
||||
middlewares = build_lead_runtime_middlewares(app_config=review_app_config)
|
||||
middlewares.append(PromptPrefixMiddleware())
|
||||
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=False,
|
||||
max_concurrent_subagents=0,
|
||||
app_config=review_app_config,
|
||||
memory_injection_enabled=False,
|
||||
)
|
||||
|
||||
agent = create_agent(
|
||||
model=model,
|
||||
tools=tools,
|
||||
middleware=middlewares,
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
|
||||
# RunnableConfig supplies thread_id so ThreadDataMiddleware is satisfied.
|
||||
import uuid
|
||||
run_config = RunnableConfig(configurable={"thread_id": f"__skill-review__{uuid.uuid4()}"})
|
||||
|
||||
# Review instructions arrive as prompt_prefix: PromptPrefixMiddleware prepends
|
||||
# them before every model call without persisting them to state.
|
||||
first_message = HumanMessage(
|
||||
content=(
|
||||
"Review this conversation and apply any skill changes you deem appropriate.\n\n"
|
||||
f"<conversation>\n{conversation_text}\n</conversation>"
|
||||
),
|
||||
additional_kwargs={"prompt_prefix": _REVIEW_SYSTEM_PROMPT},
|
||||
)
|
||||
|
||||
def _invoke_with_user_context() -> dict:
|
||||
from deerflow.runtime.user_context import set_current_user, reset_current_user
|
||||
|
||||
class _SimpleUser:
|
||||
def __init__(self, uid: str) -> None:
|
||||
self.id = uid
|
||||
|
||||
token = set_current_user(_SimpleUser(user_id))
|
||||
try:
|
||||
return agent.invoke({"messages": [first_message]}, config=run_config)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.get_event_loop().run_in_executor(None, _invoke_with_user_context),
|
||||
timeout=timeout,
|
||||
)
|
||||
# Log the final AI message so we can see what the review agent decided.
|
||||
final_msgs = (result or {}).get("messages", [])
|
||||
final_text = ""
|
||||
for msg in reversed(final_msgs):
|
||||
content = getattr(msg, "content", None)
|
||||
if content and isinstance(content, str) and content.strip():
|
||||
final_text = content.strip()[:1000]
|
||||
break
|
||||
logger.info(
|
||||
"skill_review_worker: review completed for thread %s — agent said: %s",
|
||||
thread_id,
|
||||
final_text or "(no text response)",
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning("skill_review_worker: review timed out after %ds (thread %s)", timeout, thread_id)
|
||||
except Exception:
|
||||
logger.exception("skill_review_worker: review failed for thread %s", thread_id)
|
||||
|
||||
|
||||
async def _run_review_with_pending(
|
||||
thread_id: str,
|
||||
messages: list[BaseMessage],
|
||||
app_config: "AppConfig",
|
||||
user_id: str = "default",
|
||||
) -> None:
|
||||
"""Run one review; if a newer snapshot arrived while we ran, drain it."""
|
||||
lock = _get_lock(thread_id)
|
||||
async with lock:
|
||||
await _run_review(thread_id, messages, app_config, user_id)
|
||||
|
||||
pending = _pending.pop(thread_id, None)
|
||||
if pending is not None:
|
||||
asyncio.create_task(_run_review_with_pending(thread_id, pending, app_config, user_id))
|
||||
|
||||
|
||||
def maybe_start_review(
|
||||
thread_id: str,
|
||||
messages: list[BaseMessage],
|
||||
app_config: "AppConfig",
|
||||
user_id: str = "default",
|
||||
) -> bool:
|
||||
"""Schedule a background review for ``thread_id``.
|
||||
|
||||
Returns True if a new task was started, False if queued or skipped.
|
||||
"""
|
||||
evolution_cfg = app_config.skill_evolution
|
||||
if not evolution_cfg.enabled or not evolution_cfg.review_worker.enabled:
|
||||
return False
|
||||
|
||||
tool_count = extract_tool_call_count(messages)
|
||||
if tool_count < evolution_cfg.review_worker.min_tool_calls:
|
||||
logger.debug(
|
||||
"skill_review_worker: skipping review for thread %s (%d tool calls < min %d)",
|
||||
thread_id,
|
||||
tool_count,
|
||||
evolution_cfg.review_worker.min_tool_calls,
|
||||
)
|
||||
return False
|
||||
|
||||
lock = _get_lock(thread_id)
|
||||
if lock.locked():
|
||||
_pending[thread_id] = messages
|
||||
logger.debug("skill_review_worker: queued review for thread %s (review in progress)", thread_id)
|
||||
return False
|
||||
|
||||
asyncio.create_task(_run_review_with_pending(thread_id, messages, app_config, user_id))
|
||||
logger.info("skill_review_worker: started review for thread %s (%d tool calls) user=%s", thread_id, tool_count, user_id)
|
||||
return True
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""笔记本 RAG 检索工具。
|
||||
"""空间 RAG 检索工具。
|
||||
|
||||
从 LangGraph configurable 中读取 notebook_search_space_id / notebook_source_ids /
|
||||
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在笔记本模式下引用回答。
|
||||
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在空间模式下引用回答。
|
||||
|
||||
API base URL 通过环境变量 DEER_FLOW_NOTEBOOK_API_BASE 配置。
|
||||
"""
|
||||
|
||||
+65
-20
@@ -134,27 +134,58 @@ def _default_agent_store() -> Any | None:
|
||||
return None
|
||||
|
||||
|
||||
async def _assert_agent_may_delete(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
|
||||
"""Pre-flight ownership check before agent-delete touches the on-disk file.
|
||||
async def _assert_skill_is_agent_authored(name: str) -> None:
|
||||
"""Refuse to modify skills authored by a human (UI upload / adopt / legacy).
|
||||
|
||||
LocalSkillStorage doesn't know about users — it deletes anything that
|
||||
matches by name. Without this guard, a malicious or hallucinating agent
|
||||
could destroy another user's skill just by knowing the name. We refuse
|
||||
unless the DB confirms ``user_id`` owns the skill outright.
|
||||
Even when ownership lines up, we draw a hard line at content the *human*
|
||||
curated through the Web UI: uploads, adopted orphan skills, and legacy
|
||||
skills with no usage entry (which default to ``source='upload'``).
|
||||
Those represent reviewed content the user has explicitly endorsed —
|
||||
letting the agent rewrite them risks silent drift away from what was
|
||||
approved. Only skills marked ``source='agent'`` (set by
|
||||
``mark_agent_created`` in the ``create`` flow) remain editable so the
|
||||
evolution loop can keep refining them.
|
||||
|
||||
Returns the DB record for use in the subsequent cascade step. Raises
|
||||
``ValueError`` (mapped to a tool error visible to the LLM) on refusal.
|
||||
Raises ``ValueError`` (mapped to a tool error visible to the LLM) on
|
||||
refusal.
|
||||
"""
|
||||
from deerflow.skills.usage import get_entry
|
||||
|
||||
entry = await _to_thread(get_entry, name)
|
||||
if entry.get("source") != "agent":
|
||||
raise ValueError(
|
||||
f"Refused: skill '{name}' was authored by a human via the Web UI "
|
||||
f"and cannot be modified by an agent. Ask the owner to edit it "
|
||||
f"through the Web UI, or have the user duplicate it to a new "
|
||||
f"agent-owned skill first."
|
||||
)
|
||||
|
||||
|
||||
async def _assert_agent_may_modify(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
|
||||
"""Pre-flight ownership check before an agent-driven write touches a custom skill.
|
||||
|
||||
LocalSkillStorage operates by name only — patch/edit/write_file/remove_file
|
||||
and delete all resolve a name to a directory without knowing who owns it.
|
||||
Without this guard a malicious or hallucinating agent could overwrite or
|
||||
destroy another user's skill (including UI-uploaded skills) just by
|
||||
knowing the name. We refuse unless the DB confirms ``user_id`` owns the
|
||||
skill outright.
|
||||
|
||||
Returns the DB record so callers that need owner info (e.g. delete
|
||||
cascade) can reuse it. Raises ``ValueError`` (mapped to a tool error
|
||||
visible to the LLM) on refusal.
|
||||
"""
|
||||
record = await store.get_any(name)
|
||||
if record is None:
|
||||
# No DB row — could be a legacy/orphan skill. Refuse rather than
|
||||
# delete blindly; admin can clean up via the Web UI.
|
||||
# touch blindly; users should adopt it via /api/skills/reconcile-custom
|
||||
# or an admin can clean it up via the Web UI.
|
||||
raise ValueError(
|
||||
f"Refused: skill '{name}' has no ownership record. Ask a human to delete it via the Web UI."
|
||||
f"Refused: skill '{name}' has no ownership record. Ask a human to adopt or manage it via the Web UI."
|
||||
)
|
||||
owner = record.get("owner_user_id")
|
||||
if owner is None:
|
||||
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be deleted by an agent.")
|
||||
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be modified by an agent.")
|
||||
if owner != user_id:
|
||||
raise ValueError(f"Refused: skill '{name}' is owned by another user.")
|
||||
return record
|
||||
@@ -200,6 +231,10 @@ async def _skill_manage_impl(
|
||||
lock = _get_lock(name)
|
||||
thread_id = _get_thread_id(runtime)
|
||||
skill_storage = get_or_new_skill_storage()
|
||||
# Resolved up-front so ownership checks in the write-action branches stay
|
||||
# one-liners. Cheap when the DB is unconfigured (returns None).
|
||||
user_id = get_effective_user_id()
|
||||
skill_store = _default_skill_store()
|
||||
|
||||
async with lock:
|
||||
if action == "create":
|
||||
@@ -224,14 +259,16 @@ async def _skill_manage_impl(
|
||||
# surfaces it (custom skills without a DB row are filtered out by
|
||||
# ``GET /api/skills``). Resolve the user_id *here* on the main
|
||||
# async stack — contextvars don't propagate into ``to_thread``.
|
||||
store = _default_skill_store()
|
||||
if store is not None:
|
||||
await _ensure_skill_db_registered(store, name, get_effective_user_id())
|
||||
if skill_store is not None:
|
||||
await _ensure_skill_db_registered(skill_store, name, user_id)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return f"Created custom skill '{name}'."
|
||||
|
||||
if action == "edit":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if content is None:
|
||||
raise ValueError("content is required for edit.")
|
||||
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
|
||||
@@ -250,6 +287,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "patch":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if find is None or replace is None:
|
||||
raise ValueError("find and replace are required for patch.")
|
||||
skill_file = skill_storage.get_custom_skill_file(name)
|
||||
@@ -277,11 +317,10 @@ async def _skill_manage_impl(
|
||||
# Ownership pre-check: LocalSkillStorage deletes purely by name,
|
||||
# so without this an agent could blow away another user's skill
|
||||
# just by knowing the name. Raise BEFORE touching the filesystem.
|
||||
user_id = get_effective_user_id()
|
||||
store = _default_skill_store()
|
||||
record: dict[str, Any] | None = None
|
||||
if store is not None:
|
||||
record = await _assert_agent_may_delete(store, name, user_id)
|
||||
if skill_store is not None:
|
||||
record = await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
|
||||
await _to_thread(
|
||||
skill_storage.delete_custom_skill,
|
||||
@@ -306,9 +345,9 @@ async def _skill_manage_impl(
|
||||
await _to_thread(forget, name)
|
||||
except Exception:
|
||||
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
|
||||
if store is not None:
|
||||
if skill_store is not None:
|
||||
try:
|
||||
await store.delete(name, user_id)
|
||||
await skill_store.delete(name, user_id)
|
||||
except Exception:
|
||||
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
|
||||
agent_store = _default_agent_store()
|
||||
@@ -330,6 +369,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "write_file":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if path is None or content is None:
|
||||
raise ValueError("path and content are required for write_file.")
|
||||
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
|
||||
@@ -347,6 +389,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "remove_file":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if path is None:
|
||||
raise ValueError("path is required for remove_file.")
|
||||
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
|
||||
|
||||
Reference in new issue
Block a user