feat(knowledge): 阶段二 向量 RAG 与召回注入
- embeddings.py:OpenAI 兼容 embedding 客户端,未配置时关键词兜底
- chunking.py + knowledge_embeddings 表:Markdown 分块与向量索引(Python 余弦)
- search.py:keyword/vector/hybrid 三种模式,向量不可用自动降级
- 笔记创建/更新自动重建向量索引;新增 /reindex 维护接口
- KnowledgeRagMiddleware:按对话级开关 knowledge_rag_enabled 注入 <knowledge-context>,接入 lead agent 中间件链(默认随 config.knowledge.rag_enabled)
- service.build_rag_context(wiki-query 风格分层召回)
- 笔记版本快照(knowledge_note_versions)+ GET /notes/{id}/versions
- 前端:聊天页「知识库」对话级 RAG 开关 + 类型字段
- 新增 entities/relations 表(阶段四预置)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8463289f75
commit
683cc6d76e
14 files changed
+738
-40
No files matched your search
@@ -16,6 +16,8 @@ from typing import Any
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.knowledge import extractor, llm_extractor
|
||||
from deerflow.knowledge.chunking import chunk_markdown
|
||||
from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
@@ -51,6 +53,7 @@ class KnowledgeService:
|
||||
export_html=config.export.graph_html_enabled,
|
||||
export_json=config.export.graph_json_enabled,
|
||||
)
|
||||
self.embedding_client = EmbeddingClient(config.embedding)
|
||||
try:
|
||||
self.adapter.ensure_ready()
|
||||
except Exception: # pragma: no cover - vault init best-effort
|
||||
@@ -79,8 +82,25 @@ class KnowledgeService:
|
||||
logger.exception("Failed to write knowledge note %s to vault", note_id)
|
||||
if draft.sources:
|
||||
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
|
||||
await self._reindex_embeddings(note_id, draft.content_md)
|
||||
return await self.repo.get_note(note_id) or note
|
||||
|
||||
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
|
||||
"""Best-effort: (re)build the per-chunk embedding index for a note."""
|
||||
if not self.embedding_client.is_enabled():
|
||||
return
|
||||
try:
|
||||
chunks = chunk_markdown(content_md)
|
||||
if not chunks:
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
return
|
||||
vectors = await self.embedding_client.embed_texts(chunks)
|
||||
if not vectors or len(vectors) != len(chunks):
|
||||
return
|
||||
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
|
||||
except Exception:
|
||||
logger.exception("Failed to (re)index embeddings for note %s", note_id)
|
||||
|
||||
# -- capture ----------------------------------------------------------
|
||||
async def capture_thread(
|
||||
self,
|
||||
@@ -237,6 +257,11 @@ class KnowledgeService:
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
if current is None:
|
||||
return None
|
||||
# Snapshot the pre-edit state for version history (phase 3).
|
||||
try:
|
||||
await self.repo.add_version(current, edited_by=updated_by)
|
||||
except Exception:
|
||||
logger.exception("Failed to snapshot version for note %s", note_id)
|
||||
body = content_md
|
||||
if body is not None:
|
||||
_, parsed = MarkdownVault.parse(body)
|
||||
@@ -269,8 +294,13 @@ class KnowledgeService:
|
||||
await self.repo.set_vault_path(note_id, rel)
|
||||
except Exception:
|
||||
logger.exception("Failed to update knowledge note %s in vault", note_id)
|
||||
if note.get("content_md") is not None:
|
||||
await self._reindex_embeddings(note_id, note["content_md"])
|
||||
return await self.repo.get_note(note_id)
|
||||
|
||||
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
|
||||
return await self.repo.list_versions(note_id)
|
||||
|
||||
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
|
||||
"""Soft delete (archive) by default; hard delete removes the vault file."""
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
@@ -282,13 +312,45 @@ class KnowledgeService:
|
||||
if hard and current.get("vault_path"):
|
||||
self.adapter.vault.delete_note(current["vault_path"])
|
||||
self.adapter.manifest.remove_page(note_id)
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to archive knowledge note %s in vault", note_id)
|
||||
return True
|
||||
|
||||
# -- search & graph ---------------------------------------------------
|
||||
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
|
||||
return await search_notes(self.repo, query, mode=mode, limit=limit)
|
||||
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
|
||||
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
|
||||
"""Retrieve top knowledge for ``query`` and format an injectable block.
|
||||
|
||||
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
|
||||
search when embeddings are available, else keyword (wiki-query style).
|
||||
"""
|
||||
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
|
||||
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
if not items:
|
||||
return {"text": "", "items": []}
|
||||
lines = [
|
||||
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
|
||||
"",
|
||||
]
|
||||
for i, it in enumerate(items, 1):
|
||||
lines.append(f"[知识 {i}] {it.get('title') or ''}")
|
||||
if it.get("snippet"):
|
||||
lines.append(it["snippet"])
|
||||
lines.append(f"来源:{it['note_id']}")
|
||||
lines.append("")
|
||||
return {"text": "\n".join(lines).strip(), "items": items}
|
||||
|
||||
async def reindex_all(self) -> dict[str, int]:
|
||||
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
|
||||
notes = await self.repo.list_all()
|
||||
count = 0
|
||||
for note in notes:
|
||||
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
|
||||
count += 1
|
||||
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
|
||||
|
||||
async def export_graph(self) -> dict[str, Any]:
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
Reference in new issue
Block a user