feat(knowledge): 阶段三 知识治理 + 阶段四 图谱增强
阶段三(治理):
- sensitive.py:写入前脱敏 token/密钥/手机号/身份证/邮箱(sensitive_redaction_enabled)
- 版本快照:更新前写 knowledge_note_versions;GET /notes/{id}/versions
- dedup.py:标题/摘要 Jaccard + 内容哈希查重;GET /duplicates、POST /merge(来源并入主条目、重复项归档)
- auto_ingest.py:独立后台队列/线程沉淀,不阻塞问答;中间件 after_agent 在 auto_ingest_enabled 时入队;价值判断(最小字符)+ skip_existing 去重;自动沉淀落 draft 待审
阶段四(图谱):
- LLM 抽取 entities/relations;service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边;graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据
前端:知识库页「查重合并」对话框 + 相关 api/hooks
配置:sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试:tests/test_knowledge.py 20 passed
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
683cc6d76e
commit
81e4fd92ce
16 files changed
+691
-7
No files matched your search
@@ -21,7 +21,7 @@ from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
from deerflow.knowledge.schemas import NoteDraft, normalize_source_type, normalize_status
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
|
||||
from deerflow.knowledge.search import search_notes
|
||||
from deerflow.knowledge.sources import from_reference_batches, from_search_results
|
||||
|
||||
@@ -60,8 +60,24 @@ class KnowledgeService:
|
||||
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
|
||||
|
||||
# -- internal helpers -------------------------------------------------
|
||||
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
|
||||
"""Mask secrets/PII in a draft before persistence (phase 3)."""
|
||||
if not self.config.sensitive_redaction_enabled:
|
||||
return draft
|
||||
from deerflow.knowledge.sensitive import redact
|
||||
|
||||
draft.content_md, body_hits = redact(draft.content_md)
|
||||
draft.summary, _ = redact(draft.summary)
|
||||
for src in draft.sources:
|
||||
if src.snippet:
|
||||
src.snippet, _ = redact(src.snippet)
|
||||
if body_hits:
|
||||
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
|
||||
return draft
|
||||
|
||||
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
|
||||
"""Create a DB row, write the vault file, attach sources, return note."""
|
||||
draft = self._redact_draft(draft)
|
||||
note = await self.repo.create_note(
|
||||
title=draft.title,
|
||||
summary=draft.summary,
|
||||
@@ -83,8 +99,32 @@ class KnowledgeService:
|
||||
if draft.sources:
|
||||
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
|
||||
await self._reindex_embeddings(note_id, draft.content_md)
|
||||
await self._store_entities(note_id, draft)
|
||||
return await self.repo.get_note(note_id) or note
|
||||
|
||||
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
|
||||
"""Persist extracted entities/relations and link them to the note (phase 4)."""
|
||||
if not self.config.entity_extraction_enabled or not draft.entities:
|
||||
return
|
||||
try:
|
||||
await self.repo.clear_note_relations(note_id)
|
||||
name_to_id: dict[str, str] = {}
|
||||
for ent in draft.entities:
|
||||
name = str(ent.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
row = await self.repo.get_or_create_entity(name, ent.get("type"))
|
||||
name_to_id[name.lower()] = row["id"]
|
||||
# note -> entity "mentions"
|
||||
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
|
||||
for rel in draft.relations:
|
||||
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
|
||||
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
|
||||
if a and b and a != b:
|
||||
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
|
||||
except Exception:
|
||||
logger.exception("Failed to store entities/relations for note %s", note_id)
|
||||
|
||||
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
|
||||
"""Best-effort: (re)build the per-chunk embedding index for a note."""
|
||||
if not self.embedding_client.is_enabled():
|
||||
@@ -343,6 +383,56 @@ class KnowledgeService:
|
||||
lines.append("")
|
||||
return {"text": "\n".join(lines).strip(), "items": items}
|
||||
|
||||
# -- dedup & merge (phase 3) -----------------------------------------
|
||||
async def find_duplicates(self) -> list[dict[str, Any]]:
|
||||
from deerflow.knowledge.dedup import find_duplicate_groups
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
|
||||
|
||||
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
"""Merge ``duplicate_ids`` into ``primary_id``.
|
||||
|
||||
Sources from each duplicate are re-attached to the primary, a wikilink
|
||||
+ "Merged from" note is appended to the primary body, the duplicates are
|
||||
archived (soft delete), and a ``merged_into`` relation is recorded.
|
||||
"""
|
||||
primary = await self.repo.get_note(primary_id)
|
||||
if primary is None:
|
||||
return None
|
||||
merged_titles: list[str] = []
|
||||
for dup_id in duplicate_ids:
|
||||
if dup_id == primary_id:
|
||||
continue
|
||||
dup = await self.repo.get_note(dup_id)
|
||||
if dup is None:
|
||||
continue
|
||||
# Move sources.
|
||||
for src in dup.get("sources") or []:
|
||||
await self.repo.add_sources(
|
||||
primary_id,
|
||||
[
|
||||
SourceDraft(
|
||||
source_type=src.get("source_type") or "tool",
|
||||
thread_id=src.get("thread_id"),
|
||||
message_id=src.get("message_id"),
|
||||
tool_name=src.get("tool_name"),
|
||||
title=src.get("title"),
|
||||
url=src.get("url"),
|
||||
snippet=src.get("snippet"),
|
||||
)
|
||||
],
|
||||
created_by=updated_by,
|
||||
)
|
||||
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
|
||||
await self.delete_note(dup_id, updated_by=updated_by)
|
||||
merged_titles.append(dup.get("title") or dup_id)
|
||||
|
||||
if merged_titles:
|
||||
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
|
||||
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
|
||||
return await self.repo.get_note(primary_id)
|
||||
|
||||
async def reindex_all(self) -> dict[str, int]:
|
||||
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
|
||||
notes = await self.repo.list_all()
|
||||
@@ -356,8 +446,19 @@ class KnowledgeService:
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
result = self.adapter.export_graph(notes, updated=now_iso())
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
|
||||
return {"files": result["files"], "stats": result["stats"]}
|
||||
|
||||
async def get_graph(self) -> dict[str, Any]:
|
||||
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
|
||||
from deerflow.knowledge import exporter
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
return exporter.build_graph(notes, entities, relations)
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
return self.adapter.export_file_path(filename)
|
||||
Reference in new issue
Block a user