feat: improve assistant knowledge import and database compatibility
This commit is contained in:
1 parent
fbd8dec48f
commit
6b724bd00d
51 files changed
+4393
-1021
No files matched your search
@@ -40,6 +40,7 @@ _MAX_SOURCE_CHARS = 1400
|
||||
@dataclass(frozen=True)
|
||||
class LlmWikiRagContext:
|
||||
knowledge_base_ids: list[str]
|
||||
assistant_knowledge_base_ids: list[str]
|
||||
source_count: int
|
||||
prompt: str
|
||||
sources: list[dict[str, Any]]
|
||||
@@ -50,6 +51,10 @@ def _store(request: Request) -> LlmWikiStore | None:
|
||||
return getattr(request.app.state, "llmwiki_store", None)
|
||||
|
||||
|
||||
def _assistant_store(request: Request) -> Any | None:
|
||||
return getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -76,6 +81,13 @@ def _body_selected_ids(body: Any) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def _body_selected_assistant_ids(body: Any) -> list[str]:
|
||||
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
|
||||
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
|
||||
return _as_id_list(source.get("assistant_knowledge_base_ids"))
|
||||
return []
|
||||
|
||||
|
||||
def _assistant_agent_id(body: Any) -> str | None:
|
||||
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
|
||||
if assistant_id:
|
||||
@@ -106,8 +118,17 @@ def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
|
||||
return _body_selected_ids(body)
|
||||
|
||||
|
||||
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
|
||||
"""Return assistant knowledge bases explicitly selected for the run."""
|
||||
|
||||
return _body_selected_assistant_ids(body)
|
||||
|
||||
|
||||
def llmwiki_selection_requested(body: Any) -> bool:
|
||||
return bool(resolve_requested_llmwiki_knowledge_base_ids(body))
|
||||
return bool(
|
||||
resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
or resolve_requested_assistant_knowledge_base_ids(body)
|
||||
)
|
||||
|
||||
|
||||
def _latest_human_query(graph_input: dict[str, Any]) -> str:
|
||||
@@ -191,18 +212,50 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
}
|
||||
|
||||
|
||||
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
title = str(result.get("title") or f"助手知识库片段 {index}")
|
||||
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
|
||||
content = str(
|
||||
matched_section.get("content")
|
||||
or result.get("content_markdown")
|
||||
or result.get("content")
|
||||
or result.get("summary")
|
||||
or ""
|
||||
)
|
||||
snippet = str(result.get("summary") or content)
|
||||
raw = {
|
||||
**result,
|
||||
"source": "助手知识库",
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
|
||||
"title": title,
|
||||
"type": "assistant_knowledge",
|
||||
"snippet": _truncate(snippet, 360),
|
||||
"content": content,
|
||||
"source": "助手知识库",
|
||||
"score": float(result.get("score") or 0),
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
"skillName": "assistant_knowledge_search",
|
||||
"mode": "citation",
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
if not sources:
|
||||
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
return "当前用户选择了知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
|
||||
lines = [
|
||||
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"请优先依据这些资料回答;如果资料不足,请明确说明不足之处,不要编造。",
|
||||
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
|
||||
"",
|
||||
f"用户问题:{query}",
|
||||
"",
|
||||
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
|
||||
f'<knowledge_reference_batch source_count="{len(sources)}">',
|
||||
]
|
||||
for source in sources:
|
||||
meta = " / ".join(
|
||||
@@ -213,12 +266,12 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
)
|
||||
if item
|
||||
)
|
||||
lines.append(f"[{source['index']}] {source.get('title') or 'LLMWiki 片段'}")
|
||||
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
|
||||
if meta:
|
||||
lines.append(f"来源:{meta}")
|
||||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||||
lines.append("")
|
||||
lines.append("</llmwiki_wiki_reference_batch>")
|
||||
lines.append("</knowledge_reference_batch>")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
@@ -229,52 +282,99 @@ async def build_llmwiki_rag_context(
|
||||
graph_input: dict[str, Any],
|
||||
) -> LlmWikiRagContext:
|
||||
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
if not selected_ids:
|
||||
return LlmWikiRagContext([], 0, "", [], "")
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
store = _store(request)
|
||||
if store is None:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
|
||||
if not selected_ids and not assistant_selected_ids:
|
||||
return LlmWikiRagContext([], [], 0, "", [], "")
|
||||
|
||||
query = _latest_human_query(graph_input)
|
||||
if not query:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], "")
|
||||
|
||||
user_id, is_admin = await _actor(request)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
|
||||
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
sources: list[dict[str, Any]] = []
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
return LlmWikiRagContext(selected_ids, len(sources), _build_prompt(query, sources), sources, query)
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
user_id, is_admin = await _actor(request)
|
||||
|
||||
if selected_ids:
|
||||
llmwiki_available = (local_config.enabled and local_config.internal_search_enabled) or runtime.weknora_enabled
|
||||
store = _store(request)
|
||||
if not llmwiki_available or store is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], query)
|
||||
else:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = (
|
||||
await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
if search_service is not None
|
||||
else {"results": []}
|
||||
)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning(
|
||||
"Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s",
|
||||
sanitize_log_param(user_id),
|
||||
len(rows),
|
||||
exc.code,
|
||||
)
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
payload = {"results": []}
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
if assistant_selected_ids and len(sources) < _MAX_SOURCES:
|
||||
assistant_store = _assistant_store(request)
|
||||
if assistant_store is not None:
|
||||
rows = await assistant_store.search_pages(
|
||||
base_ids=assistant_selected_ids,
|
||||
query=query,
|
||||
limit=_MAX_SOURCES - len(sources),
|
||||
include_chunks=True,
|
||||
embedding_client=getattr(request.app.state, "llmwiki_embedding", None),
|
||||
)
|
||||
for row in rows:
|
||||
sources.append(_source_from_assistant_result(row, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
len(sources),
|
||||
_build_prompt(query, sources),
|
||||
sources,
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def attach_llmwiki_rag_to_graph_input(
|
||||
@@ -294,6 +394,7 @@ def attach_llmwiki_rag_to_graph_input(
|
||||
existing = str(extra.get("prompt_prefix") or "").strip()
|
||||
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
|
||||
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
|
||||
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
|
||||
extra["llmwiki_reference_query"] = rag.query
|
||||
if rag.sources:
|
||||
extra["llmwiki_reference_sources"] = rag.sources
|
||||
|
||||
Reference in new issue
Block a user