阶段三(治理):
- sensitive.py:写入前脱敏 token/密钥/手机号/身份证/邮箱(sensitive_redaction_enabled)
- 版本快照:更新前写 knowledge_note_versions;GET /notes/{id}/versions
- dedup.py:标题/摘要 Jaccard + 内容哈希查重;GET /duplicates、POST /merge(来源并入主条目、重复项归档)
- auto_ingest.py:独立后台队列/线程沉淀,不阻塞问答;中间件 after_agent 在 auto_ingest_enabled 时入队;价值判断(最小字符)+ skip_existing 去重;自动沉淀落 draft 待审
阶段四(图谱):
- LLM 抽取 entities/relations;service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边;graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据
前端:知识库页「查重合并」对话框 + 相关 api/hooks
配置:sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试:tests/test_knowledge.py 20 passed
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
386 lines
14 KiB
Python
386 lines
14 KiB
Python
"""Knowledge base API (``/api/knowledge``).
|
|
|
|
Sediments threads / searches into an obsidian-wiki compatible vault, mirrors
|
|
metadata into the database, and serves listing, detail, edit, delete, search
|
|
and graph export. The knowledge base is **global / shared** — no per-user
|
|
isolation; writes record ``created_by`` / ``updated_by`` for audit.
|
|
|
|
All obsidian-wiki specifics live in :mod:`deerflow.knowledge`; this router only
|
|
adapts HTTP ↔ the :class:`~deerflow.knowledge.service.KnowledgeService`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import mimetypes
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, HTTPException, Request
|
|
from fastapi.responses import FileResponse
|
|
from pydantic import BaseModel, Field
|
|
|
|
from app.gateway.authz import require_auth, require_permission
|
|
from app.gateway.deps import get_checkpointer, get_knowledge_service, get_thread_store
|
|
from app.gateway.utils import sanitize_log_param
|
|
from deerflow.knowledge.service import EmptyThreadError
|
|
from deerflow.runtime import serialize_channel_values
|
|
from deerflow.runtime.references import build_reference_batches
|
|
from deerflow.runtime.user_context import get_effective_user_id
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Request / response models
|
|
# --------------------------------------------------------------------------- #
|
|
class IngestThreadRequest(BaseModel):
|
|
mode: str = Field(default="summary", description="summary | full")
|
|
title: str | None = None
|
|
status: str = Field(default="approved", description="draft | approved")
|
|
tags: list[str] = Field(default_factory=list)
|
|
include_sources: bool = True
|
|
|
|
|
|
class BatchIngestRequest(BaseModel):
|
|
limit: int = Field(default=100, ge=1, le=500)
|
|
offset: int = Field(default=0, ge=0)
|
|
status: str = "approved"
|
|
include_sources: bool = True
|
|
skip_existing: bool = True
|
|
mode: str = "summary"
|
|
|
|
|
|
class SearchResultItem(BaseModel):
|
|
title: str | None = None
|
|
url: str | None = None
|
|
snippet: str | None = None
|
|
tool_name: str | None = None
|
|
score: float | None = None
|
|
|
|
|
|
class IngestSearchRequest(BaseModel):
|
|
query: str
|
|
results: list[SearchResultItem] = Field(default_factory=list)
|
|
thread_id: str | None = None
|
|
message_id: str | None = None
|
|
title: str | None = None
|
|
tags: list[str] = Field(default_factory=list)
|
|
status: str = "approved"
|
|
|
|
|
|
class CreateNoteRequest(BaseModel):
|
|
title: str
|
|
content_md: str
|
|
summary: str | None = None
|
|
tags: list[str] = Field(default_factory=list)
|
|
status: str = "approved"
|
|
|
|
|
|
class UpdateNoteRequest(BaseModel):
|
|
title: str | None = None
|
|
content_md: str | None = None
|
|
summary: str | None = None
|
|
tags: list[str] | None = None
|
|
status: str | None = None
|
|
|
|
|
|
class SearchRequest(BaseModel):
|
|
query: str
|
|
mode: str = Field(default="keyword", description="keyword | vector | hybrid (vector/hybrid fall back to keyword in v1)")
|
|
limit: int = Field(default=8, ge=1, le=50)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Helpers
|
|
# --------------------------------------------------------------------------- #
|
|
async def _read_thread_values(checkpointer, thread_id: str) -> dict[str, Any]:
|
|
"""Return serialized channel values for a thread, or raise 404/500."""
|
|
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
|
try:
|
|
checkpoint_tuple = await checkpointer.aget_tuple(config)
|
|
except Exception:
|
|
logger.exception("Failed to read thread %s for knowledge ingest", sanitize_log_param(thread_id))
|
|
raise HTTPException(status_code=500, detail="Failed to read thread state")
|
|
if checkpoint_tuple is None:
|
|
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
|
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
|
return serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
|
|
|
|
|
def _messages_and_meta(values: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None, list[dict[str, Any]]]:
|
|
messages = values.get("messages")
|
|
if not isinstance(messages, list):
|
|
messages = []
|
|
title = values.get("title") if isinstance(values.get("title"), str) else None
|
|
batches = build_reference_batches(messages) if messages else []
|
|
return messages, title, batches
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Ingest
|
|
# --------------------------------------------------------------------------- #
|
|
@router.post("/ingest/thread/{thread_id}")
|
|
@require_auth
|
|
@require_permission("threads", "read", owner_check=True)
|
|
async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Request) -> dict[str, Any]:
|
|
"""Sediment a single thread into a knowledge note."""
|
|
service = get_knowledge_service(request)
|
|
checkpointer = get_checkpointer(request)
|
|
values = await _read_thread_values(checkpointer, thread_id)
|
|
messages, title, batches = _messages_and_meta(values)
|
|
if not messages:
|
|
raise HTTPException(status_code=400, detail="Thread has no messages to sediment")
|
|
try:
|
|
result = await service.capture_thread(
|
|
thread_id=thread_id,
|
|
messages=messages,
|
|
thread_title=title,
|
|
mode=body.mode,
|
|
title=body.title,
|
|
status=body.status,
|
|
tags=body.tags,
|
|
include_sources=body.include_sources,
|
|
reference_batches=batches,
|
|
created_by=get_effective_user_id(),
|
|
)
|
|
except EmptyThreadError:
|
|
raise HTTPException(status_code=400, detail="Thread has no sediment-able content")
|
|
return {"note": result["note"], "created": result["created"]}
|
|
|
|
|
|
@router.post("/ingest/threads/batch")
|
|
@require_auth
|
|
async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> dict[str, Any]:
|
|
"""Batch-sediment the caller's recent threads into knowledge notes."""
|
|
service = get_knowledge_service(request)
|
|
checkpointer = get_checkpointer(request)
|
|
thread_store = get_thread_store(request)
|
|
|
|
threads = await thread_store.search(limit=body.limit, offset=body.offset)
|
|
created = skipped = failed = 0
|
|
items: list[dict[str, Any]] = []
|
|
for meta in threads:
|
|
thread_id = meta.get("thread_id")
|
|
if not thread_id:
|
|
continue
|
|
try:
|
|
values = await _read_thread_values(checkpointer, thread_id)
|
|
messages, title, batches = _messages_and_meta(values)
|
|
if not messages:
|
|
skipped += 1
|
|
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
|
|
continue
|
|
result = await service.capture_thread(
|
|
thread_id=thread_id,
|
|
messages=messages,
|
|
thread_title=title or meta.get("display_name"),
|
|
mode=body.mode,
|
|
status=body.status,
|
|
include_sources=body.include_sources,
|
|
reference_batches=batches,
|
|
created_by=get_effective_user_id(),
|
|
skip_existing=body.skip_existing,
|
|
)
|
|
if result["created"]:
|
|
created += 1
|
|
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "created"})
|
|
else:
|
|
skipped += 1
|
|
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "skipped"})
|
|
except EmptyThreadError:
|
|
skipped += 1
|
|
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
|
|
except HTTPException:
|
|
failed += 1
|
|
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
|
|
except Exception:
|
|
logger.exception("Batch ingest failed for thread %s", sanitize_log_param(str(thread_id)))
|
|
failed += 1
|
|
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
|
|
|
|
return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "items": items}
|
|
|
|
|
|
@router.post("/ingest/search")
|
|
@require_auth
|
|
async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]:
|
|
"""Sediment search/tool results as a reference note."""
|
|
service = get_knowledge_service(request)
|
|
note = await service.capture_search(
|
|
query=body.query,
|
|
results=[r.model_dump() for r in body.results],
|
|
thread_id=body.thread_id,
|
|
message_id=body.message_id,
|
|
title=body.title,
|
|
tags=body.tags,
|
|
status=body.status,
|
|
created_by=get_effective_user_id(),
|
|
)
|
|
return {"note": note}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CRUD
|
|
# --------------------------------------------------------------------------- #
|
|
@router.post("/notes")
|
|
@require_auth
|
|
async def create_note(body: CreateNoteRequest, request: Request) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
note = await service.create_manual_note(
|
|
title=body.title,
|
|
content_md=body.content_md,
|
|
summary=body.summary,
|
|
tags=body.tags,
|
|
status=body.status,
|
|
created_by=get_effective_user_id(),
|
|
)
|
|
return {"note": note}
|
|
|
|
|
|
@router.get("/notes")
|
|
@require_auth
|
|
async def list_notes(
|
|
request: Request,
|
|
keyword: str | None = None,
|
|
tag: str | None = None,
|
|
source_type: str | None = None,
|
|
status: str | None = None,
|
|
limit: int = 20,
|
|
offset: int = 0,
|
|
) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
limit = max(1, min(limit, 100))
|
|
offset = max(0, offset)
|
|
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
|
|
|
|
|
|
@router.get("/notes/{note_id}")
|
|
@require_auth
|
|
async def get_note(note_id: str, request: Request) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
note = await service.get_note(note_id)
|
|
if note is None:
|
|
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
|
return note
|
|
|
|
|
|
@router.put("/notes/{note_id}")
|
|
@require_auth
|
|
async def update_note(note_id: str, body: UpdateNoteRequest, request: Request) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
note = await service.update_note(
|
|
note_id,
|
|
title=body.title,
|
|
content_md=body.content_md,
|
|
summary=body.summary,
|
|
tags=body.tags,
|
|
status=body.status,
|
|
updated_by=get_effective_user_id(),
|
|
)
|
|
if note is None:
|
|
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
|
return note
|
|
|
|
|
|
@router.delete("/notes/{note_id}")
|
|
@require_auth
|
|
async def delete_note(note_id: str, request: Request, hard: bool = False) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
ok = await service.delete_note(note_id, updated_by=get_effective_user_id(), hard=hard)
|
|
if not ok:
|
|
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
|
return {"ok": True, "note_id": note_id}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Search & graph
|
|
# --------------------------------------------------------------------------- #
|
|
@router.post("/search")
|
|
@require_auth
|
|
async def search(body: SearchRequest, request: Request) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
items = await service.search(body.query, mode=body.mode, limit=body.limit)
|
|
return {"items": items}
|
|
|
|
|
|
class MergeRequest(BaseModel):
|
|
primary_id: str
|
|
duplicate_ids: list[str] = Field(default_factory=list)
|
|
|
|
|
|
@router.get("/duplicates")
|
|
@require_auth
|
|
async def list_duplicates(request: Request) -> dict[str, Any]:
|
|
"""Return suggested duplicate groups (phase 3, heuristic)."""
|
|
service = get_knowledge_service(request)
|
|
return {"groups": await service.find_duplicates()}
|
|
|
|
|
|
@router.post("/merge")
|
|
@require_auth
|
|
async def merge_notes(body: MergeRequest, request: Request) -> dict[str, Any]:
|
|
"""Merge duplicate notes into a primary note (sources moved, dups archived)."""
|
|
service = get_knowledge_service(request)
|
|
note = await service.merge_notes(body.primary_id, body.duplicate_ids, updated_by=get_effective_user_id())
|
|
if note is None:
|
|
raise HTTPException(status_code=404, detail="Primary note not found")
|
|
return {"note": note}
|
|
|
|
|
|
@router.post("/reindex")
|
|
@require_auth
|
|
async def reindex_embeddings(request: Request) -> dict[str, Any]:
|
|
"""Rebuild the vector index for all notes (maintenance; no-op if embeddings off)."""
|
|
service = get_knowledge_service(request)
|
|
return await service.reindex_all()
|
|
|
|
|
|
@router.get("/notes/{note_id}/versions")
|
|
@require_auth
|
|
async def list_note_versions(note_id: str, request: Request) -> dict[str, Any]:
|
|
"""Return the edit history (pre-update snapshots) for a note."""
|
|
service = get_knowledge_service(request)
|
|
return {"items": await service.list_versions(note_id)}
|
|
|
|
|
|
@router.get("/graph")
|
|
@require_auth
|
|
async def get_graph(request: Request) -> dict[str, Any]:
|
|
"""Return product knowledge-graph data (notes + entities + relations)."""
|
|
service = get_knowledge_service(request)
|
|
return await service.get_graph()
|
|
|
|
|
|
@router.post("/export/graph")
|
|
@require_auth
|
|
async def export_graph(request: Request) -> dict[str, Any]:
|
|
service = get_knowledge_service(request)
|
|
result = await service.export_graph()
|
|
base = "/api/knowledge/export/files"
|
|
return {
|
|
"ok": True,
|
|
"files": {
|
|
"graph_json": f"{base}/graph.json",
|
|
"graph_html": f"{base}/graph.html",
|
|
"graph_graphml": f"{base}/graph.graphml",
|
|
"cypher": f"{base}/cypher.txt",
|
|
},
|
|
"stats": result["stats"],
|
|
}
|
|
|
|
|
|
@router.get("/export/files/{filename}")
|
|
@require_auth
|
|
async def export_file(filename: str, request: Request) -> FileResponse:
|
|
service = get_knowledge_service(request)
|
|
path = service.export_file_path(filename)
|
|
if path is None:
|
|
# Lazily build the export on first access so the iframe never 404s.
|
|
await service.export_graph()
|
|
path = service.export_file_path(filename)
|
|
if path is None:
|
|
raise HTTPException(status_code=404, detail="Export file not found")
|
|
media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
return FileResponse(path, media_type=media_type)
|