391 lines
19 KiB
Python
391 lines
19 KiB
Python
"""Knowledge-space selection helpers for normal DeerFlow chat runs.
|
||
|
||
The public chat API accepts DeerFlow-local knowledge-base ids. Selecting a
|
||
space is a scope decision, not an instruction to retrieve immediately: the
|
||
agent receives the selected ids and decides whether to call ``llmwiki_search``
|
||
for the user's actual question. This keeps unnecessary retrieval out of the
|
||
request path and leaves the tool result in the normal conversation trace.
|
||
|
||
The helper lives in ``app.gateway`` because it needs Gateway request shapes.
|
||
Harness code remains app-import free.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from collections.abc import Mapping
|
||
from dataclasses import dataclass
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from fastapi import Request
|
||
from langchain_core.messages import HumanMessage
|
||
|
||
from app.gateway.deps import get_optional_user_from_request
|
||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||
from deerflow.runtime.user_context import get_effective_user_id
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_MAX_SOURCES = 8
|
||
_MAX_SOURCE_CHARS = 1400
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class LlmWikiRagContext:
|
||
knowledge_base_ids: list[str]
|
||
assistant_knowledge_base_ids: list[str]
|
||
source_count: int
|
||
prompt: str
|
||
sources: list[dict[str, Any]]
|
||
query: str
|
||
|
||
|
||
def _store(request: Request) -> LlmWikiStore | None:
|
||
return getattr(request.app.state, "llmwiki_store", None)
|
||
|
||
|
||
def _assistant_store(request: Request) -> Any | None:
|
||
return getattr(request.app.state, "assistant_knowledge_store", None)
|
||
|
||
|
||
async def _actor(request: Request) -> tuple[str, bool]:
|
||
user = await get_optional_user_from_request(request)
|
||
if user is None:
|
||
return get_effective_user_id(), True
|
||
return str(user.id), getattr(user, "system_role", None) == "admin"
|
||
|
||
|
||
def _as_id_list(value: Any) -> list[str]:
|
||
if value is None:
|
||
return []
|
||
if isinstance(value, str):
|
||
parts = [value]
|
||
elif isinstance(value, (list, tuple, set)):
|
||
parts = list(value)
|
||
else:
|
||
return []
|
||
return list(dict.fromkeys(str(item).strip() for item in parts if str(item).strip()))
|
||
|
||
|
||
def _body_selected_ids(body: Any) -> list[str]:
|
||
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
|
||
if isinstance(source, Mapping) and "llmwiki_knowledge_base_ids" in source:
|
||
return _as_id_list(source.get("llmwiki_knowledge_base_ids"))
|
||
return []
|
||
|
||
|
||
def _body_selected_assistant_ids(body: Any) -> list[str]:
|
||
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
|
||
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
|
||
return _as_id_list(source.get("assistant_knowledge_base_ids"))
|
||
return []
|
||
|
||
|
||
def _assistant_agent_id(body: Any) -> str | None:
|
||
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
|
||
if assistant_id:
|
||
return assistant_id
|
||
context = getattr(body, "context", None)
|
||
if isinstance(context, Mapping):
|
||
return validate_agent_id(str(context.get("agent_id") or "")) if context.get("agent_id") else None
|
||
metadata = getattr(body, "metadata", None)
|
||
if isinstance(metadata, Mapping):
|
||
return validate_agent_id(str(metadata.get("agent_id") or "")) if metadata.get("agent_id") else None
|
||
return None
|
||
|
||
|
||
def _agent_bound_ids(body: Any) -> list[str]:
|
||
agent_id = _assistant_agent_id(body)
|
||
if not agent_id:
|
||
return []
|
||
try:
|
||
config = load_agent_config(agent_id)
|
||
except (FileNotFoundError, ValueError):
|
||
return []
|
||
return _as_id_list(config.llmwiki_knowledge_base_ids if config else None)
|
||
|
||
|
||
def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
|
||
"""Return knowledge bases explicitly selected for the current conversation."""
|
||
|
||
return _body_selected_ids(body)
|
||
|
||
|
||
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
|
||
"""Return assistant knowledge bases explicitly selected for the run."""
|
||
|
||
return _body_selected_assistant_ids(body)
|
||
|
||
|
||
def llmwiki_selection_requested(body: Any) -> bool:
|
||
return bool(
|
||
resolve_requested_llmwiki_knowledge_base_ids(body)
|
||
or resolve_requested_assistant_knowledge_base_ids(body)
|
||
)
|
||
|
||
|
||
def _latest_human_query(graph_input: dict[str, Any]) -> str:
|
||
messages = graph_input.get("messages")
|
||
if not isinstance(messages, list):
|
||
return ""
|
||
for message in reversed(messages):
|
||
if isinstance(message, HumanMessage):
|
||
return _message_text(getattr(message, "content", ""))
|
||
if isinstance(message, dict) and str(message.get("type") or message.get("role") or "") in {"human", "user"}:
|
||
return _message_text(message.get("content"))
|
||
return ""
|
||
|
||
|
||
def _message_text(content: Any) -> str:
|
||
if isinstance(content, str):
|
||
return content.strip()
|
||
if isinstance(content, list):
|
||
parts: list[str] = []
|
||
for item in content:
|
||
if isinstance(item, str):
|
||
parts.append(item)
|
||
elif isinstance(item, Mapping):
|
||
text = item.get("text") or item.get("content")
|
||
if isinstance(text, str):
|
||
parts.append(text)
|
||
return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
|
||
return str(content or "").strip()
|
||
|
||
|
||
def _truncate(text: str, max_chars: int) -> str:
|
||
value = text.strip()
|
||
if len(value) <= max_chars:
|
||
return value
|
||
return f"{value[: max_chars - 1].rstrip()}…"
|
||
|
||
|
||
def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index: int) -> dict[str, Any]:
|
||
if result.get("kind") == "wiki_page":
|
||
sections = list(result.get("matched_sections") or [])
|
||
content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
|
||
content = content or str(result.get("content") or result.get("content_markdown") or "")
|
||
raw = {**result, "source": mapping.get("name") or "LLMWiki"}
|
||
return {
|
||
"index": index,
|
||
"id": result.get("wiki_page_id") or result.get("id"),
|
||
"title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
|
||
"type": "llmwiki_wiki",
|
||
"snippet": _truncate(content, 360),
|
||
"content": content,
|
||
"source": mapping.get("name") or "LLMWiki",
|
||
"score": float(result.get("score") or 0),
|
||
"url": result.get("url") or "",
|
||
"skillName": "llmwiki_search",
|
||
"mode": "citation",
|
||
"raw": raw,
|
||
}
|
||
chunk_id = str(result.get("chunk_id") or "")
|
||
title = str(result.get("title") or result.get("filename") or mapping.get("name") or f"LLMWiki 片段 {index}")
|
||
content = str(result.get("content") or "")
|
||
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": chunk_id})
|
||
raw = {
|
||
**result,
|
||
"knowledge_base_id": mapping["id"],
|
||
"knowledge_base_name": mapping.get("name") or "",
|
||
"source": mapping.get("name") or "LLMWiki",
|
||
"url": f"/weknora-source-preview?{params}",
|
||
}
|
||
return {
|
||
"index": index,
|
||
"id": chunk_id or str(result.get("knowledge_id") or f"{mapping['id']}:{index}"),
|
||
"title": title,
|
||
"type": "llmwiki",
|
||
"snippet": _truncate(content, 360),
|
||
"content": content,
|
||
"source": mapping.get("name") or "LLMWiki",
|
||
"score": float(result.get("score") or 0),
|
||
"url": f"/weknora-source-preview?{params}",
|
||
"skillName": "llmwiki_search",
|
||
"mode": "citation",
|
||
"raw": raw,
|
||
}
|
||
|
||
|
||
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
|
||
title = str(result.get("title") or f"助手知识库片段 {index}")
|
||
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
|
||
content = str(
|
||
matched_section.get("content")
|
||
or result.get("content_markdown")
|
||
or result.get("content")
|
||
or result.get("summary")
|
||
or ""
|
||
)
|
||
# The retrieval citation must expose the matched knowledge itself. A page
|
||
# summary is display metadata and can omit the fact that answered the query.
|
||
snippet = content
|
||
raw = {
|
||
**result,
|
||
"source": "助手知识库",
|
||
"url": "/page/workspace/knowledge/assistant",
|
||
}
|
||
return {
|
||
"index": index,
|
||
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
|
||
"title": title,
|
||
"type": "assistant_knowledge",
|
||
"snippet": _truncate(snippet, 360),
|
||
"content": content,
|
||
"source": "助手知识库",
|
||
"score": float(result.get("score") or 0),
|
||
"url": "/page/workspace/knowledge/assistant",
|
||
"skillName": "assistant_knowledge_search",
|
||
"mode": "citation",
|
||
"raw": raw,
|
||
}
|
||
|
||
|
||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||
if not sources:
|
||
return (
|
||
"当前用户选择了知识空间,本轮只能使用所选知识空间,禁止使用 Web、MCP、技能或其他外部检索来源。"
|
||
"DeerFlow 未检索到可用片段;请直接说明所选知识空间中暂未找到相关资料,不要用外部知识补答,"
|
||
"不要编造知识库内容或引用编号。"
|
||
)
|
||
|
||
lines = [
|
||
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||
"本轮只能依据所选知识空间回答,禁止使用 Web、MCP、技能或其他外部检索来源;如果资料不足,请明确说明不足之处,不要编造或用常识补齐。",
|
||
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
|
||
"",
|
||
f"用户问题:{query}",
|
||
"",
|
||
f'<knowledge_reference_batch source_count="{len(sources)}">',
|
||
]
|
||
for source in sources:
|
||
meta = " / ".join(
|
||
str(item)
|
||
for item in (
|
||
source.get("source"),
|
||
f"score={source.get('score'):.4f}" if isinstance(source.get("score"), float) else None,
|
||
)
|
||
if item
|
||
)
|
||
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
|
||
if meta:
|
||
lines.append(f"来源:{meta}")
|
||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||
lines.append("")
|
||
lines.append("</knowledge_reference_batch>")
|
||
return "\n".join(lines).strip()
|
||
|
||
|
||
def _build_selection_prompt(knowledge_bases: list[tuple[str, str]], assistant_count: int) -> str:
|
||
"""Explain the active user-selected scope without retrieving its contents."""
|
||
|
||
lines = [
|
||
"<selected_knowledge_space>",
|
||
"本轮用户已勾选知识库。用户说“这里”“这个库”“库里”“里面”等指代时,默认就是指下列已选知识库,不要要求用户重复说明。",
|
||
]
|
||
if knowledge_bases:
|
||
lines.append("已选 LLMWiki 知识库:" + "、".join(name for _, name in knowledge_bases))
|
||
lines.append(
|
||
"调用 llmwiki_search 时必须传入 knowledge_base_id,且每次只能传下列其中一个 ID;"
|
||
"要分别比较多个库时,为每个 ID 发起一条工具调用,不能把库名仅写在 query 中:"
|
||
+ "、".join(f"{name}({mapping_id})" for mapping_id, name in knowledge_bases)
|
||
)
|
||
if knowledge_bases:
|
||
lines.append(
|
||
"HIGH-PRIORITY TOOL POLICY: llmwiki_search already returns the full content of its matched Wiki page. NEVER call llmwiki_read_wiki_page with the same knowledge_base_id + wiki_slug as a search hit; it duplicates the same information. Use llmwiki_read_wiki_page only for a DISTINCT wiki_slug found in the search hit's related_entity_slugs or inline [[slug|label]] links. Never invent a slug."
|
||
)
|
||
lines.append(
|
||
"<llmwiki_tool_workflow>\n"
|
||
"Use the two LLMWiki tools as one deliberate research workflow, not as alternatives:\n"
|
||
"1. llmwiki_search = locate the entry page and obtain its full page content, wiki_slug, and related_entity_slugs. Use that returned content as the entry overview.\n"
|
||
"1a. llmwiki_browse_wiki_index = browse a knowledge base's Wiki directory and its maintained Index introduction. This is MANDATORY as the first and ONLY LLMWiki tool call when the user asks what a library/knowledge base contains, asks for the library's summary/overview, or asks for its directory/index/catalog. Do not use llmwiki_search or llmwiki_read_wiki_page for those overview questions. If multiple knowledge bases are selected, call llmwiki_browse_wiki_index once for EACH selected knowledge_base_id, then compare their returned Index introductions and directory entries. It returns exactly one complete Wiki Index card—the same Index summary and below-summary directory content shown in the knowledge-base management UI. Only use search/page reading in a later turn when the user explicitly asks for detail about a specific topic or entry.\n"
|
||
"2. Do NOT read the search-hit page itself: llmwiki_read_wiki_page with the same knowledge_base_id + wiki_slug is forbidden because it duplicates the search result.\n"
|
||
"3. llmwiki_read_wiki_page = recursive deep-research tool for DISTINCT linked pages only. After every page read, inspect that page's in_links, out_links, and related_entity_slugs; if a directly relevant person, company, organization, concept, event, or relationship is linked, read that distinct page too. Continue this graph traversal in repeated waves until NO relevant unvisited links remain. Do not stop merely because the current evidence appears sufficient to answer the user. Prefer linked-page reads over guessing from a snippet.\n"
|
||
"4. Only after the relevant graph is exhausted may you synthesize. Cite and distinguish what each page establishes.\n"
|
||
"DETAILED KNOWLEDGE-BASE ANSWERING: selected-space questions require multi-page, multi-round research by default. Before writing a final answer, repeatedly call llmwiki_read_wiki_page to deepen the evidence. Do not say that you could continue checking, that the search result is already enough, or that you are considering another page: call the tool instead. Answer only after the relevant linked pages have actually been explored.\n"
|
||
"AVOID REDUNDANT READS: the search-hit page itself is never a valid llmwiki_read_wiki_page target in this workflow. Treat it as the entry overview and read only distinct linked pages. Never read the same knowledge_base_id + wiki_slug twice in one turn.\n"
|
||
"RECURSION SAFETY: maintain a visited set of wiki_slug values; never revisit a page, never invent a slug, and follow only links that materially help answer the question. Do not expand unrelated links merely because they exist.\n"
|
||
"</llmwiki_tool_workflow>"
|
||
)
|
||
if assistant_count:
|
||
lines.append(f"另有 {assistant_count} 个已选助手知识库。")
|
||
lines.extend(
|
||
[
|
||
"回答涉及该知识库的内容、范围、目录或事实时,先调用 llmwiki_search 检索;不要把“已选知识库”本身误当作检索结果。",
|
||
"【高优先级工具策略】本轮 llmwiki_read_wiki_page 已可用。llmwiki_search 返回任何可用 wiki_slug 后,默认应在回答前调用该工具读取页面;"
|
||
"凡是实体、概念、关系、比较、因果、范围或需要展开细节的问题必须调用。只有纯标题或一行片段即可回答的简单查询,才允许仅使用搜索片段。"
|
||
"该工具的 knowledge_base_id 必须与该检索结果的 knowledge_base_id 相同,wiki_slug 使用检索结果的 wiki_slug,或上一页返回的 in_links/out_links 中的 slug。"
|
||
"只在确有实体页/关系页可读时调用,不能虚构 wiki_slug。",
|
||
"只在本轮存在知识库勾选时适用此规则。",
|
||
"</selected_knowledge_space>",
|
||
]
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
|
||
async def build_llmwiki_rag_context(
|
||
*,
|
||
request: Request,
|
||
body: Any,
|
||
graph_input: dict[str, Any],
|
||
) -> LlmWikiRagContext:
|
||
"""Resolve selected knowledge-space ids without pre-retrieving content.
|
||
|
||
``llmwiki_search`` is deliberately agent-driven. It validates the
|
||
selection and performs the scoped local Wiki search only if the model calls
|
||
it, avoiding a duplicate search before the first model turn.
|
||
"""
|
||
# ``llmwiki_search`` is a synchronous LangChain tool that delegates its
|
||
# async retrieval work back to this Gateway event loop.
|
||
from deerflow.tools.builtins.llmwiki_search_tool import register_llmwiki_search_runtime_loop
|
||
|
||
register_llmwiki_search_runtime_loop()
|
||
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
|
||
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
|
||
selected_knowledge_bases: list[tuple[str, str]] = []
|
||
store = _store(request)
|
||
if selected_ids and store is not None:
|
||
user_id, is_admin = await _actor(request)
|
||
for mapping_id in selected_ids:
|
||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||
if row is not None:
|
||
name = str(row.get("name") or "").strip()
|
||
if name:
|
||
selected_knowledge_bases.append((str(row.get("id") or mapping_id), name[:120]))
|
||
prompt = _build_selection_prompt(selected_knowledge_bases, len(assistant_selected_ids)) if (selected_ids or assistant_selected_ids) else ""
|
||
return LlmWikiRagContext(
|
||
selected_ids,
|
||
assistant_selected_ids,
|
||
0,
|
||
prompt,
|
||
[],
|
||
_latest_human_query(graph_input),
|
||
)
|
||
|
||
|
||
def attach_llmwiki_rag_to_graph_input(
|
||
graph_input: dict[str, Any],
|
||
rag: LlmWikiRagContext,
|
||
) -> dict[str, Any]:
|
||
if not rag.prompt and not rag.sources:
|
||
return graph_input
|
||
messages = graph_input.get("messages")
|
||
if not isinstance(messages, list) or not messages:
|
||
return graph_input
|
||
new_messages = list(messages)
|
||
for index, message in enumerate(new_messages):
|
||
if not isinstance(message, HumanMessage):
|
||
continue
|
||
extra = dict(getattr(message, "additional_kwargs", None) or {})
|
||
existing = str(extra.get("prompt_prefix") or "").strip()
|
||
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
|
||
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
|
||
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
|
||
extra["llmwiki_reference_query"] = rag.query
|
||
if rag.sources:
|
||
extra["llmwiki_reference_sources"] = rag.sources
|
||
new_messages[index] = message.model_copy(update={"additional_kwargs": extra})
|
||
break
|
||
return {**graph_input, "messages": new_messages}
|