Files
deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py
T
AdministratorandClaude Opus 4.8 81e4fd92ce feat(knowledge): 阶段三 知识治理 + 阶段四 图谱增强
阶段三(治理):
- sensitive.py:写入前脱敏 token/密钥/手机号/身份证/邮箱(sensitive_redaction_enabled)
- 版本快照:更新前写 knowledge_note_versions;GET /notes/{id}/versions
- dedup.py:标题/摘要 Jaccard + 内容哈希查重;GET /duplicates、POST /merge(来源并入主条目、重复项归档)
- auto_ingest.py:独立后台队列/线程沉淀,不阻塞问答;中间件 after_agent 在 auto_ingest_enabled 时入队;价值判断(最小字符)+ skip_existing 去重;自动沉淀落 draft 待审

阶段四(图谱):
- LLM 抽取 entities/relations;service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边;graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据

前端:知识库页「查重合并」对话框 + 相关 api/hooks
配置:sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试:tests/test_knowledge.py 20 passed

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-08 18:31:33 +08:00

465 lines
20 KiB
Python

"""KnowledgeService — high-level orchestration for the knowledge base.
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
service; everything obsidian-wiki specific stays inside the ``knowledge``
module.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge import extractor, llm_extractor
from deerflow.knowledge.chunking import chunk_markdown
from deerflow.knowledge.embeddings import EmbeddingClient
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
from deerflow.knowledge.search import search_notes
from deerflow.knowledge.sources import from_reference_batches, from_search_results
logger = logging.getLogger(__name__)
class EmptyThreadError(ValueError):
"""Raised when a thread has no sediment-able content."""
def _resolve_vault_dir(vault_dir: str) -> Path:
path = Path(vault_dir)
if path.is_absolute():
return path.resolve()
return (runtime_home() / vault_dir).resolve()
class KnowledgeService:
"""Application-facing knowledge base API."""
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
self.repo = repo
self.config = config
self.project_name = config.project_name
vault_root = _resolve_vault_dir(config.vault_dir)
self.adapter = ObsidianWikiAdapter(
vault_root,
project_name=config.project_name,
export_html=config.export.graph_html_enabled,
export_json=config.export.graph_json_enabled,
)
self.embedding_client = EmbeddingClient(config.embedding)
try:
self.adapter.ensure_ready()
except Exception: # pragma: no cover - vault init best-effort
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
# -- internal helpers -------------------------------------------------
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
"""Mask secrets/PII in a draft before persistence (phase 3)."""
if not self.config.sensitive_redaction_enabled:
return draft
from deerflow.knowledge.sensitive import redact
draft.content_md, body_hits = redact(draft.content_md)
draft.summary, _ = redact(draft.summary)
for src in draft.sources:
if src.snippet:
src.snippet, _ = redact(src.snippet)
if body_hits:
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
return draft
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
draft = self._redact_draft(draft)
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
content_md=draft.content_md,
source_type=draft.source_type,
source_id=draft.source_id,
status=draft.status,
confidence=draft.confidence,
tags=draft.tags,
created_by=created_by,
)
note_id = note["id"]
try:
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
await self.repo.set_vault_path(note_id, rel)
note["vault_path"] = rel
except Exception:
logger.exception("Failed to write knowledge note %s to vault", note_id)
if draft.sources:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
await self._reindex_embeddings(note_id, draft.content_md)
await self._store_entities(note_id, draft)
return await self.repo.get_note(note_id) or note
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
"""Persist extracted entities/relations and link them to the note (phase 4)."""
if not self.config.entity_extraction_enabled or not draft.entities:
return
try:
await self.repo.clear_note_relations(note_id)
name_to_id: dict[str, str] = {}
for ent in draft.entities:
name = str(ent.get("name") or "").strip()
if not name:
continue
row = await self.repo.get_or_create_entity(name, ent.get("type"))
name_to_id[name.lower()] = row["id"]
# note -> entity "mentions"
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
for rel in draft.relations:
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
if a and b and a != b:
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
except Exception:
logger.exception("Failed to store entities/relations for note %s", note_id)
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
"""Best-effort: (re)build the per-chunk embedding index for a note."""
if not self.embedding_client.is_enabled():
return
try:
chunks = chunk_markdown(content_md)
if not chunks:
await self.repo.delete_embeddings(note_id)
return
vectors = await self.embedding_client.embed_texts(chunks)
if not vectors or len(vectors) != len(chunks):
return
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
except Exception:
logger.exception("Failed to (re)index embeddings for note %s", note_id)
# -- capture ----------------------------------------------------------
async def capture_thread(
self,
*,
thread_id: str,
messages: list[dict[str, Any]],
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
status: str = "approved",
tags: list[str] | None = None,
include_sources: bool = True,
reference_batches: list[dict[str, Any]] | None = None,
created_by: str | None = None,
skip_existing: bool = False,
) -> dict[str, Any]:
"""Sediment a thread into a knowledge note.
Returns ``{"note": <note>, "created": bool}``. Raises
:class:`EmptyThreadError` when there is nothing worth keeping.
"""
if skip_existing:
existing = await self.repo.find_by_source("thread", thread_id)
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
existing = await self.repo.find_by_source("thread", thread_id)
if existing:
return {"note": existing, "created": False}
sources = []
if include_sources and reference_batches:
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
draft: NoteDraft | None = None
# Prefer LLM distillation (wiki-capture style) so notes are declarative
# knowledge, not raw chat. Falls back to the rule-based extractor when
# the LLM is disabled/unavailable or returns something unusable.
if self.config.llm_extract_enabled:
draft = await llm_extractor.extract_thread_llm(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
model_name=self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
)
if draft is None:
draft = extractor.extract_thread(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
)
if draft is None:
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
draft.status = normalize_status(status)
note = await self._persist_draft(draft, created_by=created_by)
return {"note": note, "created": True}
async def capture_search(
self,
*,
query: str,
results: list[dict[str, Any]] | None,
thread_id: str | None = None,
message_id: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
) -> dict[str, Any]:
"""Sediment search/tool results into a reference note."""
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
draft.status = normalize_status(status)
return await self._persist_draft(draft, created_by=created_by)
async def create_manual_note(
self,
*,
title: str,
content_md: str,
tags: list[str] | None = None,
status: str = "approved",
summary: str | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
"""Create a note directly from product-edited Markdown."""
# Strip any frontmatter the user may have pasted; vault writer adds its own.
_, body = MarkdownVault.parse(content_md)
body = body or content_md
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
draft = NoteDraft(
title=title.strip() or "知识笔记",
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
content_md=body,
category="synthesis",
source_type="manual",
source_id=None,
status=normalize_status(status),
tags=base_tags,
confidence=0.5,
related=extractor.related_links(self.project_name),
)
return await self._persist_draft(draft, created_by=created_by)
# -- CRUD -------------------------------------------------------------
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
offset: int = 0,
) -> dict[str, Any]:
items, total = await self.repo.list_notes(
keyword=keyword,
tag=tag,
source_type=normalize_source_type(source_type) if source_type else None,
status=status,
limit=limit,
offset=offset,
)
return {"items": items, "total": total}
async def get_note(self, note_id: str) -> dict[str, Any] | None:
return await self.repo.get_note(note_id)
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
content_md: str | None = None,
tags: list[str] | None = None,
status: str | None = None,
summary: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any] | None:
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return None
# Snapshot the pre-edit state for version history (phase 3).
try:
await self.repo.add_version(current, edited_by=updated_by)
except Exception:
logger.exception("Failed to snapshot version for note %s", note_id)
body = content_md
if body is not None:
_, parsed = MarkdownVault.parse(body)
body = parsed or body
note = await self.repo.update_note(
note_id,
title=title,
summary=summary,
content_md=body,
status=normalize_status(status) if status else None,
tags=tags,
updated_by=updated_by,
)
if note is None:
return None
try:
rel = self.adapter.update_note_file(
note_id,
current.get("vault_path"),
title=note["title"],
content_md=note["content_md"],
tags=note.get("tags") or [],
status=note["status"],
summary=note.get("summary") or "",
category="synthesis",
created=note["created_at"],
updated=note["updated_at"],
)
if rel and rel != current.get("vault_path"):
await self.repo.set_vault_path(note_id, rel)
except Exception:
logger.exception("Failed to update knowledge note %s in vault", note_id)
if note.get("content_md") is not None:
await self._reindex_embeddings(note_id, note["content_md"])
return await self.repo.get_note(note_id)
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
return await self.repo.list_versions(note_id)
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
"""Soft delete (archive) by default; hard delete removes the vault file."""
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return False
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
try:
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
if hard and current.get("vault_path"):
self.adapter.vault.delete_note(current["vault_path"])
self.adapter.manifest.remove_page(note_id)
await self.repo.delete_embeddings(note_id)
except Exception:
logger.exception("Failed to archive knowledge note %s in vault", note_id)
return True
# -- search & graph ---------------------------------------------------
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
"""Retrieve top knowledge for ``query`` and format an injectable block.
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
search when embeddings are available, else keyword (wiki-query style).
"""
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
if not items:
return {"text": "", "items": []}
lines = [
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
"",
]
for i, it in enumerate(items, 1):
lines.append(f"[知识 {i}] {it.get('title') or ''}")
if it.get("snippet"):
lines.append(it["snippet"])
lines.append(f"来源:{it['note_id']}")
lines.append("")
return {"text": "\n".join(lines).strip(), "items": items}
# -- dedup & merge (phase 3) -----------------------------------------
async def find_duplicates(self) -> list[dict[str, Any]]:
from deerflow.knowledge.dedup import find_duplicate_groups
notes = await self.repo.list_all()
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
"""Merge ``duplicate_ids`` into ``primary_id``.
Sources from each duplicate are re-attached to the primary, a wikilink
+ "Merged from" note is appended to the primary body, the duplicates are
archived (soft delete), and a ``merged_into`` relation is recorded.
"""
primary = await self.repo.get_note(primary_id)
if primary is None:
return None
merged_titles: list[str] = []
for dup_id in duplicate_ids:
if dup_id == primary_id:
continue
dup = await self.repo.get_note(dup_id)
if dup is None:
continue
# Move sources.
for src in dup.get("sources") or []:
await self.repo.add_sources(
primary_id,
[
SourceDraft(
source_type=src.get("source_type") or "tool",
thread_id=src.get("thread_id"),
message_id=src.get("message_id"),
tool_name=src.get("tool_name"),
title=src.get("title"),
url=src.get("url"),
snippet=src.get("snippet"),
)
],
created_by=updated_by,
)
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
await self.delete_note(dup_id, updated_by=updated_by)
merged_titles.append(dup.get("title") or dup_id)
if merged_titles:
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
return await self.repo.get_note(primary_id)
async def reindex_all(self) -> dict[str, int]:
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
notes = await self.repo.list_all()
count = 0
for note in notes:
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
count += 1
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
async def export_graph(self) -> dict[str, Any]:
from deerflow.utils.time import now_iso
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
return {"files": result["files"], "stats": result["stats"]}
async def get_graph(self) -> dict[str, Any]:
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
from deerflow.knowledge import exporter
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
return exporter.build_graph(notes, entities, relations)
def export_file_path(self, filename: str) -> Path | None:
return self.adapter.export_file_path(filename)