feat: add skill and assistant knowledge bases

This commit is contained in:
Administrator committed 2026-09-02 00:49:13 +08:00
1 parent dd0ee282df
commit fbd8dec48f
45 files changed
+5262 -21

No files matched your search

@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,143 @@
"""ORM model for the single global DeerFlow assistant knowledge base."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,601 @@
"""Repository for the single global assistant knowledge base."""
from __future__ import annotations
import difflib
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
return data
class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_base(self) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
existing = await self.get_base()
if existing:
return existing
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
async with self._sf() as session:
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def _upsert_source(
self,
session: AsyncSession,
*,
base_id: str,
source_type: str,
source_key: str,
source_name: str,
metadata: dict,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> AssistantKnowledgeSourceRow:
stmt = select(AssistantKnowledgeSourceRow).where(
AssistantKnowledgeSourceRow.base_id == base_id,
AssistantKnowledgeSourceRow.source_type == source_type,
AssistantKnowledgeSourceRow.source_key == source_key,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = AssistantKnowledgeSourceRow(
id=str(uuid4()),
base_id=base_id,
source_type=source_type,
source_key=source_key,
source_name=source_name,
skill_name=skill_name,
knowledge_base_mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
status="active",
metadata_json=metadata,
created_at=_now(),
updated_at=_now(),
)
session.add(row)
await session.flush()
else:
row.source_name = source_name
row.status = "active"
row.metadata_json = metadata
row.updated_at = _now()
return row
async def _publish_page(
self,
session: AsyncSession,
*,
base_id: str,
slug: str,
title: str,
page_type: str,
content: str,
summary: str,
search_terms: list[str],
source_id: str,
import_job_id: str,
created_by: str | None,
change_type: str,
) -> AssistantKnowledgeWikiPageRow:
now = _now()
page = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base_id,
AssistantKnowledgeWikiPageRow.slug == slug,
)
)
).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base_id,
slug=slug,
title=title,
page_type=page_type,
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source_id,
import_job_id=import_job_id,
change_type=change_type,
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.page_type = page_type
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, *search_terms])
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
return page
async def import_package(
self,
*,
package: dict[str, Any],
source_type: str,
source_key: str,
source_name: str,
trigger: str,
created_by: str | None,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> dict[str, Any]:
base = await self.initialize(created_by=created_by)
now = _now()
async with self._sf() as session:
source = await self._upsert_source(
session,
base_id=base["id"],
source_type=source_type,
source_key=source_key,
source_name=source_name,
metadata=package.get("source") or {},
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
# A re-import replaces this source's prior contributions while retaining
# canonical entities/relations that are still supported by other sources.
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
job = AssistantKnowledgeImportJobRow(
id=str(uuid4()),
base_id=base["id"],
source_id=source.id,
trigger=trigger,
status="running",
phase="aligning_entities",
counts_json={},
created_by=created_by,
created_at=now,
)
session.add(job)
await session.flush()
source.last_import_job_id = job.id
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
raw_entities = list(package.get("entities") or [])
for raw in raw_entities:
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
normalized = normalize_name(display_name)
if not normalized:
continue
stmt = select(AssistantKnowledgeEntityRow).where(
AssistantKnowledgeEntityRow.base_id == base["id"],
AssistantKnowledgeEntityRow.entity_type == entity_type,
AssistantKnowledgeEntityRow.normalized_name == normalized,
)
entity = (await session.execute(stmt)).scalar_one_or_none()
if entity is None:
entity = AssistantKnowledgeEntityRow(
id=str(uuid4()),
base_id=base["id"],
entity_type=entity_type,
normalized_name=normalized,
display_name=display_name,
aliases_json=raw.get("aliases") or [],
attributes_json=raw.get("attributes") or {},
status="active",
)
session.add(entity)
await session.flush()
raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
entity_map[raw_id] = entity.id
entity_rows[entity.id] = (entity, raw)
session.add(
AssistantKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=entity.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
source_chunk_id=raw.get("source_chunk_id"),
source_skill_name=skill_name,
evidence_json=raw.get("evidence") or [],
confidence=float(raw.get("confidence") or 1.0),
)
)
raw_relations = list(package.get("relations") or [])
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
for raw in raw_relations:
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
confidence = float(raw.get("confidence") or 1.0)
if not source_entity_id or not target_entity_id or confidence < 0.75:
continue
predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
stmt = select(AssistantKnowledgeRelationRow).where(
AssistantKnowledgeRelationRow.base_id == base["id"],
AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
AssistantKnowledgeRelationRow.predicate == predicate,
AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
)
relation = (await session.execute(stmt)).scalar_one_or_none()
if relation is None:
relation = AssistantKnowledgeRelationRow(
id=str(uuid4()),
base_id=base["id"],
source_entity_id=source_entity_id,
predicate=predicate,
target_entity_id=target_entity_id,
description=str(raw.get("description") or ""),
status="active",
)
session.add(relation)
await session.flush()
relation_rows[relation.id] = (relation, raw)
session.add(
AssistantKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=relation.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
graph_edge_id=raw.get("graph_edge_id"),
evidence_json=raw.get("evidence") or [],
confidence=confidence,
)
)
for entity, raw in entity_rows.values():
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
entity.aliases_json = aliases
content = "\n".join(
[
f"# {entity.display_name}",
"",
f"- 类型:{entity.entity_type}",
f"- 别名:{'、'.join(aliases) if aliases else '无'}",
f"- 本次来源:{source_name}",
"",
"## 属性",
"",
"```json",
str(entity.attributes_json or {}),
"```",
]
)
entity_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"entities/{entity.entity_type}/{entity.id}",
title=entity.display_name,
page_type="entity",
content=content,
summary=f"{entity.entity_type} 实体,来源:{source_name}",
search_terms=aliases,
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="entity_alignment",
)
entity.wiki_page_id = entity_page.id
for relation, raw in relation_rows.values():
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源:{source_name}\n- 置信度:{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
relation_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"relations/{relation.id}",
title=f"{source_name_display} {relation.predicate} {target_name_display}",
page_type="relation",
content=content,
summary=relation.description,
search_terms=[source_name_display, target_name_display, relation.predicate],
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="relation_alignment",
)
relation.wiki_page_id = relation_page.id
pages = list(package.get("wiki_pages") or [])
if not pages and package.get("wiki"):
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
pages = [page] if page else []
for raw in pages:
if not isinstance(raw, dict):
continue
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
title = str(raw.get("title") or source_name)
content = str(raw.get("content") or raw.get("content_markdown") or "")
summary = str(raw.get("summary") or "")
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
page = (await session.execute(stmt)).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base["id"],
slug=slug,
title=title,
page_type=str(raw.get("page_type") or "source"),
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source.id,
import_job_id=job.id,
change_type="skill_sync" if source_type == "skill" else "import",
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
page.current_revision_id = revision.id
page.updated_at = now
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"documents": len(package.get("documents") or []),
"chunks": len(package.get("chunks") or []),
"entities": len(raw_entities),
"relations": len(raw_relations),
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
"graph_edges": len((package.get("graph") or {}).get("edges") or []),
}
job.status = "completed"
job.phase = "completed"
job.counts_json = counts
job.completed_at = now
await self._retire_orphans(session, base["id"])
await session.flush()
result = _dict(job)
await session.commit()
return result
async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
for entity in entities:
contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
for relation in relations:
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
if query:
needle = f"%{query.strip()}%"
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
if page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_page(self, slug: str) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
if page is None:
return []
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
return _dict(row) if row else None
async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
left, right = await self.get_revision(left_id), await self.get_revision(right_id)
if not left or not right or left["page_id"] != right["page_id"]:
return None
diff = "\n".join(
difflib.unified_diff(
str(left["content_markdown"]).splitlines(),
str(right["content_markdown"]).splitlines(),
fromfile=f"revision-{left['revision_no']}",
tofile=f"revision-{right['revision_no']}",
lineterm="",
)
)
return {"left": left, "right": right, "diff": diff}
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
now = _now()
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
if page is None or source is None or source.page_id != page.id:
return None
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
row = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=source.content_markdown,
summary=source.summary,
source_id=source.source_id,
import_job_id=source.import_job_id,
change_type="rollback",
status="published",
created_by=actor,
created_at=now,
published_at=now,
)
session.add(row)
page.current_revision_id = row.id
page.content_markdown = row.content_markdown
page.summary = row.summary
page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
page.updated_at = now
await session.flush()
data = _dict(row)
await session.commit()
return data
async def list_sources(self) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeSourceRow, source_id)
return _dict(row) if row and row.status != "deleted" else None
async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeEntityRow, entity_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeRelationRow, relation_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeImportJobRow, job_id)
return _dict(row) if row else None
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
source = await session.get(AssistantKnowledgeSourceRow, source_id)
if source is None:
return False
source.status = "deleted"
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
pages = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
)
).scalars()
for page in pages:
fallback = (
await session.execute(
select(AssistantKnowledgeWikiRevisionRow)
.where(
AssistantKnowledgeWikiRevisionRow.page_id == page.id,
AssistantKnowledgeWikiRevisionRow.source_id != source_id,
AssistantKnowledgeWikiRevisionRow.status == "published",
)
.order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
.limit(1)
)
).scalar_one_or_none()
if fallback is None:
page.status = "deleted"
page.current_revision_id = None
else:
page.current_revision_id = fallback.id
page.content_markdown = fallback.content_markdown
page.summary = fallback.summary
page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
await self._retire_orphans(session, source.base_id)
await session.commit()
return True
def make_assistant_knowledge_store(session_factory):
return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None