feat: add skill and assistant knowledge bases
This commit is contained in:
1 parent
dd0ee282df
commit
fbd8dec48f
45 files changed
+5262
-21
No files matched your search
+4
@@ -0,0 +1,4 @@
|
||||
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
|
||||
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""ORM model for the single global DeerFlow assistant knowledge base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class AssistantKnowledgeBaseRow(Base):
|
||||
__tablename__ = "assistant_knowledge_base"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeSourceRow(Base):
|
||||
__tablename__ = "assistant_knowledge_sources"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
source_key: Mapped[str] = mapped_column(String(255))
|
||||
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
source_name: Mapped[str] = mapped_column(String(255), default="")
|
||||
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeImportJobRow(Base):
|
||||
__tablename__ = "assistant_knowledge_import_jobs"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
phase: Mapped[str] = mapped_column(String(32), default="exporting")
|
||||
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiPageRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_pages"
|
||||
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
slug: Mapped[str] = mapped_column(String(512), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), index=True)
|
||||
page_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiRevisionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_revisions"
|
||||
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
revision_no: Mapped[int] = mapped_column(Integer)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
change_type: Mapped[str] = mapped_column(String(32), default="import")
|
||||
status: Mapped[str] = mapped_column(String(24), default="published")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
entity_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
normalized_name: Mapped[str] = mapped_column(String(512))
|
||||
display_name: Mapped[str] = mapped_column(String(512), index=True)
|
||||
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entity_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relations"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
predicate: Mapped[str] = mapped_column(String(128), index=True)
|
||||
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relation_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
relation_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
+601
@@ -0,0 +1,601 @@
|
||||
"""Repository for the single global assistant knowledge base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
AssistantKnowledgeImportJobRow,
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row: Any) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key, value in tuple(data.items()):
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
elif key.endswith("_json"):
|
||||
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
|
||||
return data
|
||||
|
||||
|
||||
class AssistantKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_base(self) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
|
||||
existing = await self.get_base()
|
||||
if existing:
|
||||
return existing
|
||||
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def _upsert_source(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
base_id: str,
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
metadata: dict,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
) -> AssistantKnowledgeSourceRow:
|
||||
stmt = select(AssistantKnowledgeSourceRow).where(
|
||||
AssistantKnowledgeSourceRow.base_id == base_id,
|
||||
AssistantKnowledgeSourceRow.source_type == source_type,
|
||||
AssistantKnowledgeSourceRow.source_key == source_key,
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = AssistantKnowledgeSourceRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base_id,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
skill_name=skill_name,
|
||||
knowledge_base_mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
status="active",
|
||||
metadata_json=metadata,
|
||||
created_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
else:
|
||||
row.source_name = source_name
|
||||
row.status = "active"
|
||||
row.metadata_json = metadata
|
||||
row.updated_at = _now()
|
||||
return row
|
||||
|
||||
async def _publish_page(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
base_id: str,
|
||||
slug: str,
|
||||
title: str,
|
||||
page_type: str,
|
||||
content: str,
|
||||
summary: str,
|
||||
search_terms: list[str],
|
||||
source_id: str,
|
||||
import_job_id: str,
|
||||
created_by: str | None,
|
||||
change_type: str,
|
||||
) -> AssistantKnowledgeWikiPageRow:
|
||||
now = _now()
|
||||
page = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base_id,
|
||||
AssistantKnowledgeWikiPageRow.slug == slug,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if page is None:
|
||||
page = AssistantKnowledgeWikiPageRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base_id,
|
||||
slug=slug,
|
||||
title=title,
|
||||
page_type=page_type,
|
||||
content_markdown="",
|
||||
summary="",
|
||||
search_text="",
|
||||
status="active",
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(page)
|
||||
await session.flush()
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
|
||||
revision = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=content,
|
||||
summary=summary,
|
||||
source_id=source_id,
|
||||
import_job_id=import_job_id,
|
||||
change_type=change_type,
|
||||
status="published",
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(revision)
|
||||
page.title = title
|
||||
page.page_type = page_type
|
||||
page.content_markdown = content
|
||||
page.summary = summary
|
||||
page.search_text = "\n".join([title, summary, content, *search_terms])
|
||||
page.current_revision_id = revision.id
|
||||
page.status = "active"
|
||||
page.updated_at = now
|
||||
return page
|
||||
|
||||
async def import_package(
|
||||
self,
|
||||
*,
|
||||
package: dict[str, Any],
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
trigger: str,
|
||||
created_by: str | None,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
base = await self.initialize(created_by=created_by)
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
source = await self._upsert_source(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
metadata=package.get("source") or {},
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
# A re-import replaces this source's prior contributions while retaining
|
||||
# canonical entities/relations that are still supported by other sources.
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
|
||||
job = AssistantKnowledgeImportJobRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_id=source.id,
|
||||
trigger=trigger,
|
||||
status="running",
|
||||
phase="aligning_entities",
|
||||
counts_json={},
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
source.last_import_job_id = job.id
|
||||
|
||||
entity_map: dict[str, str] = {}
|
||||
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
|
||||
raw_entities = list(package.get("entities") or [])
|
||||
for raw in raw_entities:
|
||||
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
|
||||
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
|
||||
normalized = normalize_name(display_name)
|
||||
if not normalized:
|
||||
continue
|
||||
stmt = select(AssistantKnowledgeEntityRow).where(
|
||||
AssistantKnowledgeEntityRow.base_id == base["id"],
|
||||
AssistantKnowledgeEntityRow.entity_type == entity_type,
|
||||
AssistantKnowledgeEntityRow.normalized_name == normalized,
|
||||
)
|
||||
entity = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if entity is None:
|
||||
entity = AssistantKnowledgeEntityRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
entity_type=entity_type,
|
||||
normalized_name=normalized,
|
||||
display_name=display_name,
|
||||
aliases_json=raw.get("aliases") or [],
|
||||
attributes_json=raw.get("attributes") or {},
|
||||
status="active",
|
||||
)
|
||||
session.add(entity)
|
||||
await session.flush()
|
||||
raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
|
||||
entity_map[raw_id] = entity.id
|
||||
entity_rows[entity.id] = (entity, raw)
|
||||
session.add(
|
||||
AssistantKnowledgeEntityContributionRow(
|
||||
id=str(uuid4()),
|
||||
entity_id=entity.id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_wiki_slug=raw.get("source_wiki_slug"),
|
||||
source_chunk_id=raw.get("source_chunk_id"),
|
||||
source_skill_name=skill_name,
|
||||
evidence_json=raw.get("evidence") or [],
|
||||
confidence=float(raw.get("confidence") or 1.0),
|
||||
)
|
||||
)
|
||||
|
||||
raw_relations = list(package.get("relations") or [])
|
||||
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
|
||||
for raw in raw_relations:
|
||||
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
|
||||
target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
|
||||
confidence = float(raw.get("confidence") or 1.0)
|
||||
if not source_entity_id or not target_entity_id or confidence < 0.75:
|
||||
continue
|
||||
predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
|
||||
stmt = select(AssistantKnowledgeRelationRow).where(
|
||||
AssistantKnowledgeRelationRow.base_id == base["id"],
|
||||
AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
|
||||
AssistantKnowledgeRelationRow.predicate == predicate,
|
||||
AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
|
||||
)
|
||||
relation = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if relation is None:
|
||||
relation = AssistantKnowledgeRelationRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_entity_id=source_entity_id,
|
||||
predicate=predicate,
|
||||
target_entity_id=target_entity_id,
|
||||
description=str(raw.get("description") or ""),
|
||||
status="active",
|
||||
)
|
||||
session.add(relation)
|
||||
await session.flush()
|
||||
relation_rows[relation.id] = (relation, raw)
|
||||
session.add(
|
||||
AssistantKnowledgeRelationContributionRow(
|
||||
id=str(uuid4()),
|
||||
relation_id=relation.id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_wiki_slug=raw.get("source_wiki_slug"),
|
||||
graph_edge_id=raw.get("graph_edge_id"),
|
||||
evidence_json=raw.get("evidence") or [],
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
|
||||
for entity, raw in entity_rows.values():
|
||||
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
|
||||
entity.aliases_json = aliases
|
||||
content = "\n".join(
|
||||
[
|
||||
f"# {entity.display_name}",
|
||||
"",
|
||||
f"- 类型:{entity.entity_type}",
|
||||
f"- 别名:{'、'.join(aliases) if aliases else '无'}",
|
||||
f"- 本次来源:{source_name}",
|
||||
"",
|
||||
"## 属性",
|
||||
"",
|
||||
"```json",
|
||||
str(entity.attributes_json or {}),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
entity_page = await self._publish_page(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
slug=f"entities/{entity.entity_type}/{entity.id}",
|
||||
title=entity.display_name,
|
||||
page_type="entity",
|
||||
content=content,
|
||||
summary=f"{entity.entity_type} 实体,来源:{source_name}",
|
||||
search_terms=aliases,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
created_by=created_by,
|
||||
change_type="entity_alignment",
|
||||
)
|
||||
entity.wiki_page_id = entity_page.id
|
||||
|
||||
for relation, raw in relation_rows.values():
|
||||
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
|
||||
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
|
||||
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
|
||||
target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
|
||||
content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源:{source_name}\n- 置信度:{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
|
||||
relation_page = await self._publish_page(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
slug=f"relations/{relation.id}",
|
||||
title=f"{source_name_display} {relation.predicate} {target_name_display}",
|
||||
page_type="relation",
|
||||
content=content,
|
||||
summary=relation.description,
|
||||
search_terms=[source_name_display, target_name_display, relation.predicate],
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
created_by=created_by,
|
||||
change_type="relation_alignment",
|
||||
)
|
||||
relation.wiki_page_id = relation_page.id
|
||||
|
||||
pages = list(package.get("wiki_pages") or [])
|
||||
if not pages and package.get("wiki"):
|
||||
wiki = package["wiki"]
|
||||
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
|
||||
pages = [page] if page else []
|
||||
for raw in pages:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
|
||||
title = str(raw.get("title") or source_name)
|
||||
content = str(raw.get("content") or raw.get("content_markdown") or "")
|
||||
summary = str(raw.get("summary") or "")
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
|
||||
page = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if page is None:
|
||||
page = AssistantKnowledgeWikiPageRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
slug=slug,
|
||||
title=title,
|
||||
page_type=str(raw.get("page_type") or "source"),
|
||||
content_markdown="",
|
||||
summary="",
|
||||
search_text="",
|
||||
status="active",
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(page)
|
||||
await session.flush()
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
|
||||
revision = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=content,
|
||||
summary=summary,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
change_type="skill_sync" if source_type == "skill" else "import",
|
||||
status="published",
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(revision)
|
||||
page.title = title
|
||||
page.content_markdown = content
|
||||
page.summary = summary
|
||||
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
|
||||
page.current_revision_id = revision.id
|
||||
page.updated_at = now
|
||||
|
||||
counts = {
|
||||
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
|
||||
"documents": len(package.get("documents") or []),
|
||||
"chunks": len(package.get("chunks") or []),
|
||||
"entities": len(raw_entities),
|
||||
"relations": len(raw_relations),
|
||||
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
|
||||
"graph_edges": len((package.get("graph") or {}).get("edges") or []),
|
||||
}
|
||||
job.status = "completed"
|
||||
job.phase = "completed"
|
||||
job.counts_json = counts
|
||||
job.completed_at = now
|
||||
await self._retire_orphans(session, base["id"])
|
||||
await session.flush()
|
||||
result = _dict(job)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
|
||||
entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
|
||||
for entity in entities:
|
||||
contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
|
||||
entity.status = "active" if contribution else "deleted"
|
||||
relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
|
||||
for relation in relations:
|
||||
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
|
||||
relation.status = "active" if contribution else "deleted"
|
||||
|
||||
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
|
||||
if query:
|
||||
needle = f"%{query.strip()}%"
|
||||
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
|
||||
if page_type:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_page(self, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
if page is None:
|
||||
return []
|
||||
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
|
||||
left, right = await self.get_revision(left_id), await self.get_revision(right_id)
|
||||
if not left or not right or left["page_id"] != right["page_id"]:
|
||||
return None
|
||||
diff = "\n".join(
|
||||
difflib.unified_diff(
|
||||
str(left["content_markdown"]).splitlines(),
|
||||
str(right["content_markdown"]).splitlines(),
|
||||
fromfile=f"revision-{left['revision_no']}",
|
||||
tofile=f"revision-{right['revision_no']}",
|
||||
lineterm="",
|
||||
)
|
||||
)
|
||||
return {"left": left, "right": right, "diff": diff}
|
||||
|
||||
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
|
||||
if page is None or source is None or source.page_id != page.id:
|
||||
return None
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
|
||||
row = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=source.content_markdown,
|
||||
summary=source.summary,
|
||||
source_id=source.source_id,
|
||||
import_job_id=source.import_job_id,
|
||||
change_type="rollback",
|
||||
status="published",
|
||||
created_by=actor,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
page.current_revision_id = row.id
|
||||
page.content_markdown = row.content_markdown
|
||||
page.summary = row.summary
|
||||
page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
|
||||
page.updated_at = now
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def list_sources(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeSourceRow, source_id)
|
||||
return _dict(row) if row and row.status != "deleted" else None
|
||||
|
||||
async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeEntityRow, entity_id)
|
||||
if row is None or row.status != "active":
|
||||
return None
|
||||
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
|
||||
result = _dict(row)
|
||||
result["contributions"] = [_dict(value) for value in contributions]
|
||||
return result
|
||||
|
||||
async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeRelationRow, relation_id)
|
||||
if row is None or row.status != "active":
|
||||
return None
|
||||
contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
|
||||
result = _dict(row)
|
||||
result["contributions"] = [_dict(value) for value in contributions]
|
||||
return result
|
||||
|
||||
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeImportJobRow, job_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
source = await session.get(AssistantKnowledgeSourceRow, source_id)
|
||||
if source is None:
|
||||
return False
|
||||
source.status = "deleted"
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
|
||||
pages = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
|
||||
)
|
||||
).scalars()
|
||||
for page in pages:
|
||||
fallback = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiRevisionRow)
|
||||
.where(
|
||||
AssistantKnowledgeWikiRevisionRow.page_id == page.id,
|
||||
AssistantKnowledgeWikiRevisionRow.source_id != source_id,
|
||||
AssistantKnowledgeWikiRevisionRow.status == "published",
|
||||
)
|
||||
.order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
page.status = "deleted"
|
||||
page.current_revision_id = None
|
||||
else:
|
||||
page.current_revision_id = fallback.id
|
||||
page.content_markdown = fallback.content_markdown
|
||||
page.summary = fallback.summary
|
||||
page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
|
||||
await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
|
||||
await self._retire_orphans(session, source.base_id)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
def make_assistant_knowledge_store(session_factory):
|
||||
return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None
|
||||
Reference in new issue
Block a user