feat(knowledge): 阶段三 知识治理 + 阶段四 图谱增强

阶段三(治理):
- sensitive.py:写入前脱敏 token/密钥/手机号/身份证/邮箱(sensitive_redaction_enabled)
- 版本快照:更新前写 knowledge_note_versions;GET /notes/{id}/versions
- dedup.py:标题/摘要 Jaccard + 内容哈希查重;GET /duplicates、POST /merge(来源并入主条目、重复项归档)
- auto_ingest.py:独立后台队列/线程沉淀,不阻塞问答;中间件 after_agent 在 auto_ingest_enabled 时入队;价值判断(最小字符)+ skip_existing 去重;自动沉淀落 draft 待审

阶段四(图谱):
- LLM 抽取 entities/relations;service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边;graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据

前端:知识库页「查重合并」对话框 + 相关 api/hooks
配置:sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试:tests/test_knowledge.py 20 passed

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-08 18:31:33 +08:00
1 parent 683cc6d76e
commit 81e4fd92ce
16 files changed
+691 -7

No files matched your search

@@ -21,7 +21,7 @@ from deerflow.knowledge.embeddings import EmbeddingClient
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, normalize_source_type, normalize_status
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
from deerflow.knowledge.search import search_notes
from deerflow.knowledge.sources import from_reference_batches, from_search_results
@@ -60,8 +60,24 @@ class KnowledgeService:
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
# -- internal helpers -------------------------------------------------
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
"""Mask secrets/PII in a draft before persistence (phase 3)."""
if not self.config.sensitive_redaction_enabled:
return draft
from deerflow.knowledge.sensitive import redact
draft.content_md, body_hits = redact(draft.content_md)
draft.summary, _ = redact(draft.summary)
for src in draft.sources:
if src.snippet:
src.snippet, _ = redact(src.snippet)
if body_hits:
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
return draft
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
draft = self._redact_draft(draft)
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
@@ -83,8 +99,32 @@ class KnowledgeService:
if draft.sources:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
await self._reindex_embeddings(note_id, draft.content_md)
await self._store_entities(note_id, draft)
return await self.repo.get_note(note_id) or note
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
"""Persist extracted entities/relations and link them to the note (phase 4)."""
if not self.config.entity_extraction_enabled or not draft.entities:
return
try:
await self.repo.clear_note_relations(note_id)
name_to_id: dict[str, str] = {}
for ent in draft.entities:
name = str(ent.get("name") or "").strip()
if not name:
continue
row = await self.repo.get_or_create_entity(name, ent.get("type"))
name_to_id[name.lower()] = row["id"]
# note -> entity "mentions"
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
for rel in draft.relations:
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
if a and b and a != b:
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
except Exception:
logger.exception("Failed to store entities/relations for note %s", note_id)
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
"""Best-effort: (re)build the per-chunk embedding index for a note."""
if not self.embedding_client.is_enabled():
@@ -343,6 +383,56 @@ class KnowledgeService:
lines.append("")
return {"text": "\n".join(lines).strip(), "items": items}
# -- dedup & merge (phase 3) -----------------------------------------
async def find_duplicates(self) -> list[dict[str, Any]]:
from deerflow.knowledge.dedup import find_duplicate_groups
notes = await self.repo.list_all()
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
"""Merge ``duplicate_ids`` into ``primary_id``.
Sources from each duplicate are re-attached to the primary, a wikilink
+ "Merged from" note is appended to the primary body, the duplicates are
archived (soft delete), and a ``merged_into`` relation is recorded.
"""
primary = await self.repo.get_note(primary_id)
if primary is None:
return None
merged_titles: list[str] = []
for dup_id in duplicate_ids:
if dup_id == primary_id:
continue
dup = await self.repo.get_note(dup_id)
if dup is None:
continue
# Move sources.
for src in dup.get("sources") or []:
await self.repo.add_sources(
primary_id,
[
SourceDraft(
source_type=src.get("source_type") or "tool",
thread_id=src.get("thread_id"),
message_id=src.get("message_id"),
tool_name=src.get("tool_name"),
title=src.get("title"),
url=src.get("url"),
snippet=src.get("snippet"),
)
],
created_by=updated_by,
)
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
await self.delete_note(dup_id, updated_by=updated_by)
merged_titles.append(dup.get("title") or dup_id)
if merged_titles:
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
return await self.repo.get_note(primary_id)
async def reindex_all(self) -> dict[str, int]:
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
notes = await self.repo.list_all()
@@ -356,8 +446,19 @@ class KnowledgeService:
from deerflow.utils.time import now_iso
notes = await self.repo.list_all()
result = self.adapter.export_graph(notes, updated=now_iso())
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
return {"files": result["files"], "stats": result["stats"]}
async def get_graph(self) -> dict[str, Any]:
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
from deerflow.knowledge import exporter
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
return exporter.build_graph(notes, entities, relations)
def export_file_path(self, filename: str) -> Path | None:
return self.adapter.export_file_path(filename)