AdministratorandClaude Opus 4.8
cc99ebba81
后端：新增自包含 deerflow.knowledge 模块（models/repository/schemas/ extractor/sources/markdown_vault/manifest/obsidian_adapter/exporter/ search/service），通过 deps.py 装配为 app.state.knowledge_service， 挂载 /api/knowledge 路由；config.yaml 增加 knowledge 配置段， AppConfig 注册 KnowledgeConfig，ORM 注册 4 张 knowledge_* 表。 前端：新增知识库页面（KnowledgeBasePage/KnowledgeGraphPage）、 core/knowledge api/hooks/types、聊天侧 knowledge-save-trigger， 侧边栏增加「知识库」入口。 知识库为全局共享，不做 user_id 隔离，写入记录 created_by/updated_by 作为审计。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
316 lines
12 KiB
Python
316 lines
12 KiB
Python
"""Database read/write for the knowledge base.
|
|
|
|
This layer never calls an LLM and never touches the Markdown vault — it only
|
|
persists/queries the relational mirror (notes, sources, tags). The vault and
|
|
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
|
|
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import delete, func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from deerflow.knowledge.models import (
|
|
KnowledgeNoteRow,
|
|
KnowledgeNoteTagRow,
|
|
KnowledgeSourceRow,
|
|
KnowledgeTagRow,
|
|
)
|
|
from deerflow.knowledge.schemas import SourceDraft
|
|
|
|
|
|
def _new_id(prefix: str) -> str:
|
|
return f"{prefix}_{uuid.uuid4().hex[:24]}"
|
|
|
|
|
|
def _iso(value: Any) -> Any:
|
|
if isinstance(value, datetime):
|
|
return value.isoformat()
|
|
return value
|
|
|
|
|
|
class KnowledgeRepository:
|
|
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
|
|
|
|
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
|
self._sf = session_factory
|
|
|
|
# -- serialization ----------------------------------------------------
|
|
@staticmethod
|
|
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
|
|
d = row.to_dict()
|
|
d["created_at"] = _iso(d.get("created_at"))
|
|
d["updated_at"] = _iso(d.get("updated_at"))
|
|
return d
|
|
|
|
@staticmethod
|
|
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
|
|
d = row.to_dict()
|
|
d["created_at"] = _iso(d.get("created_at"))
|
|
return d
|
|
|
|
# -- tags -------------------------------------------------------------
|
|
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
|
|
cleaned = []
|
|
seen: set[str] = set()
|
|
for raw in names or []:
|
|
name = (raw or "").strip()
|
|
if not name or name.lower() in seen:
|
|
continue
|
|
seen.add(name.lower())
|
|
cleaned.append(name)
|
|
if not cleaned:
|
|
return []
|
|
existing = {
|
|
r.name.lower(): r
|
|
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
|
|
}
|
|
result: list[KnowledgeTagRow] = []
|
|
for name in cleaned:
|
|
row = existing.get(name.lower())
|
|
if row is None:
|
|
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
|
|
session.add(row)
|
|
await session.flush()
|
|
existing[name.lower()] = row
|
|
result.append(row)
|
|
return result
|
|
|
|
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
|
|
stmt = (
|
|
select(KnowledgeTagRow.name)
|
|
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
|
.where(KnowledgeNoteTagRow.note_id == note_id)
|
|
.order_by(KnowledgeTagRow.name)
|
|
)
|
|
return list((await session.execute(stmt)).scalars())
|
|
|
|
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
|
|
if not note_ids:
|
|
return {}
|
|
stmt = (
|
|
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
|
|
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
|
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
|
|
.order_by(KnowledgeTagRow.name)
|
|
)
|
|
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
|
|
for note_id, name in (await session.execute(stmt)).all():
|
|
out.setdefault(note_id, []).append(name)
|
|
return out
|
|
|
|
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
|
|
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
|
|
tags = await self._get_or_create_tags(session, names)
|
|
for tag in tags:
|
|
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
|
|
|
|
# -- notes ------------------------------------------------------------
|
|
async def create_note(
|
|
self,
|
|
*,
|
|
title: str,
|
|
summary: str | None,
|
|
content_md: str,
|
|
source_type: str,
|
|
source_id: str | None,
|
|
status: str,
|
|
confidence: float = 0.0,
|
|
vault_path: str | None = None,
|
|
tags: list[str] | None = None,
|
|
created_by: str | None = None,
|
|
) -> dict[str, Any]:
|
|
note_id = _new_id("kb")
|
|
now = datetime.now(UTC)
|
|
row = KnowledgeNoteRow(
|
|
id=note_id,
|
|
title=title[:512],
|
|
summary=summary,
|
|
content_md=content_md,
|
|
vault_path=vault_path,
|
|
source_type=source_type,
|
|
source_id=source_id,
|
|
status=status,
|
|
confidence=confidence,
|
|
created_by=created_by,
|
|
updated_by=created_by,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
async with self._sf() as session:
|
|
session.add(row)
|
|
await session.flush()
|
|
await self._replace_note_tags(session, note_id, tags or [])
|
|
await session.commit()
|
|
await session.refresh(row)
|
|
d = self._note_to_dict(row)
|
|
d["tags"] = await self._tags_for(session, note_id)
|
|
return d
|
|
|
|
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
|
|
async with self._sf() as session:
|
|
row = await session.get(KnowledgeNoteRow, note_id)
|
|
if row is None:
|
|
return
|
|
row.vault_path = vault_path
|
|
await session.commit()
|
|
|
|
async def update_note(
|
|
self,
|
|
note_id: str,
|
|
*,
|
|
title: str | None = None,
|
|
summary: str | None = None,
|
|
content_md: str | None = None,
|
|
status: str | None = None,
|
|
tags: list[str] | None = None,
|
|
vault_path: str | None = None,
|
|
updated_by: str | None = None,
|
|
) -> dict[str, Any] | None:
|
|
async with self._sf() as session:
|
|
row = await session.get(KnowledgeNoteRow, note_id)
|
|
if row is None:
|
|
return None
|
|
if title is not None:
|
|
row.title = title[:512]
|
|
if summary is not None:
|
|
row.summary = summary
|
|
if content_md is not None:
|
|
row.content_md = content_md
|
|
if status is not None:
|
|
row.status = status
|
|
if vault_path is not None:
|
|
row.vault_path = vault_path
|
|
if updated_by is not None:
|
|
row.updated_by = updated_by
|
|
row.updated_at = datetime.now(UTC)
|
|
if tags is not None:
|
|
await self._replace_note_tags(session, note_id, tags)
|
|
await session.commit()
|
|
await session.refresh(row)
|
|
d = self._note_to_dict(row)
|
|
d["tags"] = await self._tags_for(session, note_id)
|
|
return d
|
|
|
|
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
|
|
return await self.update_note(note_id, status=status, updated_by=updated_by)
|
|
|
|
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
|
|
async with self._sf() as session:
|
|
row = await session.get(KnowledgeNoteRow, note_id)
|
|
if row is None:
|
|
return None
|
|
d = self._note_to_dict(row)
|
|
d["tags"] = await self._tags_for(session, note_id)
|
|
if with_sources:
|
|
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
|
|
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
|
|
return d
|
|
|
|
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
|
|
"""Return the first (non-archived) note for a (source_type, source_id)."""
|
|
async with self._sf() as session:
|
|
stmt = (
|
|
select(KnowledgeNoteRow)
|
|
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
|
|
.where(KnowledgeNoteRow.status != "archived")
|
|
.limit(1)
|
|
)
|
|
row = (await session.execute(stmt)).scalars().first()
|
|
return self._note_to_dict(row) if row else None
|
|
|
|
async def list_notes(
|
|
self,
|
|
*,
|
|
keyword: str | None = None,
|
|
tag: str | None = None,
|
|
source_type: str | None = None,
|
|
status: str | None = None,
|
|
include_archived: bool = False,
|
|
limit: int = 20,
|
|
offset: int = 0,
|
|
) -> tuple[list[dict[str, Any]], int]:
|
|
async with self._sf() as session:
|
|
stmt = select(KnowledgeNoteRow)
|
|
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
|
|
|
|
conditions = []
|
|
if status:
|
|
conditions.append(KnowledgeNoteRow.status == status)
|
|
elif not include_archived:
|
|
conditions.append(KnowledgeNoteRow.status != "archived")
|
|
if source_type:
|
|
conditions.append(KnowledgeNoteRow.source_type == source_type)
|
|
if keyword:
|
|
like = f"%{keyword}%"
|
|
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
|
|
if tag:
|
|
tag_subq = (
|
|
select(KnowledgeNoteTagRow.note_id)
|
|
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
|
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
|
|
)
|
|
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
|
|
|
|
for cond in conditions:
|
|
stmt = stmt.where(cond)
|
|
count_stmt = count_stmt.where(cond)
|
|
|
|
total = (await session.execute(count_stmt)).scalar_one()
|
|
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
|
|
rows = list((await session.execute(stmt)).scalars())
|
|
ids = [r.id for r in rows]
|
|
tags_map = await self._tags_for_many(session, ids)
|
|
items = []
|
|
for r in rows:
|
|
d = self._note_to_dict(r)
|
|
d["tags"] = tags_map.get(r.id, [])
|
|
items.append(d)
|
|
return items, int(total)
|
|
|
|
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
|
|
"""Return every note (light fields + tags) for graph export."""
|
|
async with self._sf() as session:
|
|
stmt = select(KnowledgeNoteRow)
|
|
if not include_archived:
|
|
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
|
|
rows = list((await session.execute(stmt)).scalars())
|
|
tags_map = await self._tags_for_many(session, [r.id for r in rows])
|
|
out = []
|
|
for r in rows:
|
|
d = self._note_to_dict(r)
|
|
d["tags"] = tags_map.get(r.id, [])
|
|
out.append(d)
|
|
return out
|
|
|
|
# -- sources ----------------------------------------------------------
|
|
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
|
|
if not sources:
|
|
return
|
|
import json
|
|
|
|
async with self._sf() as session:
|
|
for src in sources:
|
|
session.add(
|
|
KnowledgeSourceRow(
|
|
id=_new_id("kbsrc"),
|
|
note_id=note_id,
|
|
source_type=src.source_type,
|
|
thread_id=src.thread_id,
|
|
message_id=src.message_id,
|
|
tool_name=src.tool_name,
|
|
title=(src.title or "")[:512] or None,
|
|
url=src.url,
|
|
snippet=src.snippet,
|
|
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
|
|
created_by=created_by,
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
)
|
|
await session.commit()
|