feat: add skill and assistant knowledge bases

This commit is contained in:
Administrator committed 2026-09-02 00:49:13 +08:00
1 parent dd0ee282df
commit fbd8dec48f
45 files changed
+5262 -21

No files matched your search

@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,143 @@
"""ORM model for the single global DeerFlow assistant knowledge base."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,601 @@
"""Repository for the single global assistant knowledge base."""
from __future__ import annotations
import difflib
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
return data
class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_base(self) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
existing = await self.get_base()
if existing:
return existing
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
async with self._sf() as session:
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def _upsert_source(
self,
session: AsyncSession,
*,
base_id: str,
source_type: str,
source_key: str,
source_name: str,
metadata: dict,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> AssistantKnowledgeSourceRow:
stmt = select(AssistantKnowledgeSourceRow).where(
AssistantKnowledgeSourceRow.base_id == base_id,
AssistantKnowledgeSourceRow.source_type == source_type,
AssistantKnowledgeSourceRow.source_key == source_key,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = AssistantKnowledgeSourceRow(
id=str(uuid4()),
base_id=base_id,
source_type=source_type,
source_key=source_key,
source_name=source_name,
skill_name=skill_name,
knowledge_base_mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
status="active",
metadata_json=metadata,
created_at=_now(),
updated_at=_now(),
)
session.add(row)
await session.flush()
else:
row.source_name = source_name
row.status = "active"
row.metadata_json = metadata
row.updated_at = _now()
return row
async def _publish_page(
self,
session: AsyncSession,
*,
base_id: str,
slug: str,
title: str,
page_type: str,
content: str,
summary: str,
search_terms: list[str],
source_id: str,
import_job_id: str,
created_by: str | None,
change_type: str,
) -> AssistantKnowledgeWikiPageRow:
now = _now()
page = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base_id,
AssistantKnowledgeWikiPageRow.slug == slug,
)
)
).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base_id,
slug=slug,
title=title,
page_type=page_type,
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source_id,
import_job_id=import_job_id,
change_type=change_type,
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.page_type = page_type
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, *search_terms])
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
return page
async def import_package(
self,
*,
package: dict[str, Any],
source_type: str,
source_key: str,
source_name: str,
trigger: str,
created_by: str | None,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> dict[str, Any]:
base = await self.initialize(created_by=created_by)
now = _now()
async with self._sf() as session:
source = await self._upsert_source(
session,
base_id=base["id"],
source_type=source_type,
source_key=source_key,
source_name=source_name,
metadata=package.get("source") or {},
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
# A re-import replaces this source's prior contributions while retaining
# canonical entities/relations that are still supported by other sources.
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
job = AssistantKnowledgeImportJobRow(
id=str(uuid4()),
base_id=base["id"],
source_id=source.id,
trigger=trigger,
status="running",
phase="aligning_entities",
counts_json={},
created_by=created_by,
created_at=now,
)
session.add(job)
await session.flush()
source.last_import_job_id = job.id
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
raw_entities = list(package.get("entities") or [])
for raw in raw_entities:
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
normalized = normalize_name(display_name)
if not normalized:
continue
stmt = select(AssistantKnowledgeEntityRow).where(
AssistantKnowledgeEntityRow.base_id == base["id"],
AssistantKnowledgeEntityRow.entity_type == entity_type,
AssistantKnowledgeEntityRow.normalized_name == normalized,
)
entity = (await session.execute(stmt)).scalar_one_or_none()
if entity is None:
entity = AssistantKnowledgeEntityRow(
id=str(uuid4()),
base_id=base["id"],
entity_type=entity_type,
normalized_name=normalized,
display_name=display_name,
aliases_json=raw.get("aliases") or [],
attributes_json=raw.get("attributes") or {},
status="active",
)
session.add(entity)
await session.flush()
raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
entity_map[raw_id] = entity.id
entity_rows[entity.id] = (entity, raw)
session.add(
AssistantKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=entity.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
source_chunk_id=raw.get("source_chunk_id"),
source_skill_name=skill_name,
evidence_json=raw.get("evidence") or [],
confidence=float(raw.get("confidence") or 1.0),
)
)
raw_relations = list(package.get("relations") or [])
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
for raw in raw_relations:
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
confidence = float(raw.get("confidence") or 1.0)
if not source_entity_id or not target_entity_id or confidence < 0.75:
continue
predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
stmt = select(AssistantKnowledgeRelationRow).where(
AssistantKnowledgeRelationRow.base_id == base["id"],
AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
AssistantKnowledgeRelationRow.predicate == predicate,
AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
)
relation = (await session.execute(stmt)).scalar_one_or_none()
if relation is None:
relation = AssistantKnowledgeRelationRow(
id=str(uuid4()),
base_id=base["id"],
source_entity_id=source_entity_id,
predicate=predicate,
target_entity_id=target_entity_id,
description=str(raw.get("description") or ""),
status="active",
)
session.add(relation)
await session.flush()
relation_rows[relation.id] = (relation, raw)
session.add(
AssistantKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=relation.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
graph_edge_id=raw.get("graph_edge_id"),
evidence_json=raw.get("evidence") or [],
confidence=confidence,
)
)
for entity, raw in entity_rows.values():
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
entity.aliases_json = aliases
content = "\n".join(
[
f"# {entity.display_name}",
"",
f"- 类型:{entity.entity_type}",
f"- 别名:{'、'.join(aliases) if aliases else '无'}",
f"- 本次来源:{source_name}",
"",
"## 属性",
"",
"```json",
str(entity.attributes_json or {}),
"```",
]
)
entity_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"entities/{entity.entity_type}/{entity.id}",
title=entity.display_name,
page_type="entity",
content=content,
summary=f"{entity.entity_type} 实体,来源:{source_name}",
search_terms=aliases,
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="entity_alignment",
)
entity.wiki_page_id = entity_page.id
for relation, raw in relation_rows.values():
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源:{source_name}\n- 置信度:{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
relation_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"relations/{relation.id}",
title=f"{source_name_display} {relation.predicate} {target_name_display}",
page_type="relation",
content=content,
summary=relation.description,
search_terms=[source_name_display, target_name_display, relation.predicate],
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="relation_alignment",
)
relation.wiki_page_id = relation_page.id
pages = list(package.get("wiki_pages") or [])
if not pages and package.get("wiki"):
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
pages = [page] if page else []
for raw in pages:
if not isinstance(raw, dict):
continue
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
title = str(raw.get("title") or source_name)
content = str(raw.get("content") or raw.get("content_markdown") or "")
summary = str(raw.get("summary") or "")
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
page = (await session.execute(stmt)).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base["id"],
slug=slug,
title=title,
page_type=str(raw.get("page_type") or "source"),
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source.id,
import_job_id=job.id,
change_type="skill_sync" if source_type == "skill" else "import",
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
page.current_revision_id = revision.id
page.updated_at = now
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"documents": len(package.get("documents") or []),
"chunks": len(package.get("chunks") or []),
"entities": len(raw_entities),
"relations": len(raw_relations),
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
"graph_edges": len((package.get("graph") or {}).get("edges") or []),
}
job.status = "completed"
job.phase = "completed"
job.counts_json = counts
job.completed_at = now
await self._retire_orphans(session, base["id"])
await session.flush()
result = _dict(job)
await session.commit()
return result
async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
for entity in entities:
contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
for relation in relations:
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
if query:
needle = f"%{query.strip()}%"
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
if page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_page(self, slug: str) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
if page is None:
return []
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
return _dict(row) if row else None
async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
left, right = await self.get_revision(left_id), await self.get_revision(right_id)
if not left or not right or left["page_id"] != right["page_id"]:
return None
diff = "\n".join(
difflib.unified_diff(
str(left["content_markdown"]).splitlines(),
str(right["content_markdown"]).splitlines(),
fromfile=f"revision-{left['revision_no']}",
tofile=f"revision-{right['revision_no']}",
lineterm="",
)
)
return {"left": left, "right": right, "diff": diff}
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
now = _now()
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
if page is None or source is None or source.page_id != page.id:
return None
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
row = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=source.content_markdown,
summary=source.summary,
source_id=source.source_id,
import_job_id=source.import_job_id,
change_type="rollback",
status="published",
created_by=actor,
created_at=now,
published_at=now,
)
session.add(row)
page.current_revision_id = row.id
page.content_markdown = row.content_markdown
page.summary = row.summary
page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
page.updated_at = now
await session.flush()
data = _dict(row)
await session.commit()
return data
async def list_sources(self) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeSourceRow, source_id)
return _dict(row) if row and row.status != "deleted" else None
async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeEntityRow, entity_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeRelationRow, relation_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeImportJobRow, job_id)
return _dict(row) if row else None
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
source = await session.get(AssistantKnowledgeSourceRow, source_id)
if source is None:
return False
source.status = "deleted"
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
pages = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
)
).scalars()
for page in pages:
fallback = (
await session.execute(
select(AssistantKnowledgeWikiRevisionRow)
.where(
AssistantKnowledgeWikiRevisionRow.page_id == page.id,
AssistantKnowledgeWikiRevisionRow.source_id != source_id,
AssistantKnowledgeWikiRevisionRow.status == "published",
)
.order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
.limit(1)
)
).scalar_one_or_none()
if fallback is None:
page.status = "deleted"
page.current_revision_id = None
else:
page.current_revision_id = fallback.id
page.content_markdown = fallback.content_markdown
page.summary = fallback.summary
page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
await self._retire_orphans(session, source.base_id)
await session.commit()
return True
def make_assistant_knowledge_store(session_factory):
return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -0,0 +1,73 @@
"""Add skill distillation and assistant knowledge tables.
Revision ID: 20260901_01
Revises: 20260825_01
Create Date: 2026-09-01
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
revision = "20260901_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
TABLES = [
SkillKnowledgeBindingRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncJobRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeArtifactRow,
SkillKnowledgeReviewRow,
SkillKnowledgeEntityRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeRelationContributionRow,
AssistantKnowledgeBaseRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeRelationContributionRow,
]
def upgrade() -> None:
bind = op.get_bind()
for model in TABLES:
model.__table__.create(bind, checkfirst=True)
def downgrade() -> None:
bind = op.get_bind()
for model in reversed(TABLES):
model.__table__.drop(bind, checkfirst=True)
@@ -35,6 +35,17 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -86,6 +97,18 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -107,6 +130,15 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
"AssistantKnowledgeEntityRow",
"AssistantKnowledgeEntityContributionRow",
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
@@ -164,6 +196,16 @@ __all__ = [
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
"SkillKnowledgeArtifactRow",
"SkillKnowledgeBindingRow",
"SkillKnowledgeEntityContributionRow",
"SkillKnowledgeEntityRow",
"SkillKnowledgeRelationContributionRow",
"SkillKnowledgeRelationRow",
"SkillKnowledgeReviewRow",
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
@@ -0,0 +1,15 @@
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
@@ -0,0 +1,203 @@
"""ORM rows for skill-to-knowledge distillation and review."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class SkillKnowledgeBindingRow(Base):
__tablename__ = "skill_knowledge_bindings"
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
target_id: Mapped[str] = mapped_column(String(128), index=True)
target_name: Mapped[str] = mapped_column(String(255), default="")
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeSnapshotRow(Base):
__tablename__ = "skill_knowledge_snapshots"
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
source_digest: Mapped[str] = mapped_column(String(80), index=True)
extractor_version: Mapped[str] = mapped_column(String(64))
prompt_hash: Mapped[str] = mapped_column(String(80))
model_name: Mapped[str] = mapped_column(String(128))
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="ready")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeSyncJobRow(Base):
__tablename__ = "skill_knowledge_sync_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
trigger: Mapped[str] = mapped_column(String(32), default="manual")
target_count: Mapped[int] = mapped_column(Integer, default=0)
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
total_count: Mapped[int] = mapped_column(Integer, default=0)
success_count: Mapped[int] = mapped_column(Integer, default=0)
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
failed_count: Mapped[int] = mapped_column(Integer, default=0)
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeSyncItemRow(Base):
__tablename__ = "skill_knowledge_sync_items"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
target_type: Mapped[str] = mapped_column(String(32))
target_mode: Mapped[str] = mapped_column(String(32))
target_id: Mapped[str] = mapped_column(String(128))
skill_name: Mapped[str] = mapped_column(String(255), index=True)
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="scanning")
progress: Mapped[int] = mapped_column(Integer, default=0)
attempts: Mapped[int] = mapped_column(Integer, default=0)
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeArtifactRow(Base):
__tablename__ = "skill_knowledge_artifacts"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
kind: Mapped[str] = mapped_column(String(48), index=True)
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeReviewRow(Base):
__tablename__ = "skill_knowledge_reviews"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
item_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
object_type: Mapped[str] = mapped_column(String(32), index=True)
object_key: Mapped[str] = mapped_column(String(128))
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeEntityRow(Base):
__tablename__ = "skill_knowledge_entities"
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512))
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeEntityContributionRow(Base):
__tablename__ = "skill_knowledge_entity_contributions"
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_entity_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class SkillKnowledgeRelationRow(Base):
__tablename__ = "skill_knowledge_relations"
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeRelationContributionRow(Base):
__tablename__ = "skill_knowledge_relation_contributions"
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_relation_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,758 @@
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
list_json_fields = {
"aliases_json",
"evidence_json",
"manifest_json",
"refs_json",
"targets_json",
}
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
return data
class SkillKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
entities = (
await session.execute(
select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == target_type,
SkillKnowledgeEntityRow.target_id == target_id,
)
)
).scalars()
for entity in entities:
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (
await session.execute(
select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == target_type,
SkillKnowledgeRelationRow.target_id == target_id,
)
)
).scalars()
for relation in relations:
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def create_job(
self,
*,
skill_names: list[str],
targets: list[dict[str, Any]],
created_by: str | None,
force: bool = False,
trigger: str = "manual",
review_mode: str = "auto_high_confidence",
) -> dict[str, Any]:
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
normalized_targets: list[dict[str, Any]] = []
seen_targets: set[tuple[str, str]] = set()
for target in targets:
target_type = str(target.get("target_type") or "").strip()
target_id = str(target.get("target_id") or "").strip()
if not target_type or not target_id or (target_type, target_id) in seen_targets:
continue
seen_targets.add((target_type, target_id))
normalized_targets.append(
{
"target_type": target_type,
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
"target_id": target_id,
"target_name": str(target.get("target_name") or target_id),
"remote_business_key": target.get("remote_business_key"),
"readonly": bool(target.get("readonly", False)),
}
)
job_id = str(uuid4())
now = _now()
async with self._sf() as session:
job = SkillKnowledgeSyncJobRow(
id=job_id,
trigger=trigger,
target_count=len(normalized_targets),
targets_json=normalized_targets,
status="queued",
total_count=0,
review_mode=review_mode,
created_by=created_by,
created_at=now,
)
session.add(job)
for skill_name in names:
for target in normalized_targets:
stmt = select(SkillKnowledgeBindingRow).where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.target_type == target["target_type"],
SkillKnowledgeBindingRow.target_id == target["target_id"],
)
binding = (await session.execute(stmt)).scalar_one_or_none()
if binding is None:
binding = SkillKnowledgeBindingRow(
id=str(uuid4()),
skill_name=skill_name,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
target_name=target["target_name"],
remote_business_key=target.get("remote_business_key"),
status="queued",
created_by=created_by,
created_at=now,
updated_at=now,
)
session.add(binding)
else:
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
continue
binding.target_mode = target["target_mode"]
binding.target_name = target["target_name"]
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
job_id=job_id,
binding_id=binding.id,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
skill_name=skill_name,
status="queued",
phase="scanning",
progress=0,
active_dedupe_key=binding.id,
created_at=now,
updated_at=now,
)
session.add(item)
binding.last_job_id = job_id
binding.last_item_id = item.id
job.total_count += 1
if job.total_count == 0:
job.status = "completed"
job.completed_at = now
await session.flush()
result = _dict(job)
result["force"] = force
await session.commit()
return result
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
if target_id:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
return _dict(row) if row else None
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
if active_only:
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def has_active_item(self, binding_id: str) -> bool:
async with self._sf() as session:
value = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
return value is not None
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
if row is None or not row.active:
return None
row.auto_sync_enabled = enabled
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return None
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
result = _dict(job)
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()
stmt = (
select(SkillKnowledgeSyncItemRow)
.where(
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
)
)
.order_by(SkillKnowledgeSyncItemRow.created_at)
.limit(limit)
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def claim_item(
self,
item_id: str,
*,
owner: str,
lease_seconds: int = 900,
) -> dict[str, Any] | None:
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
now = _now()
stmt = (
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
),
)
.values(
status="running",
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
lease_owner=owner,
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
async with self._sf() as session:
result = await session.execute(stmt)
if result.rowcount != 1:
await session.rollback()
return None
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
SkillKnowledgeSyncItemRow.status == "running",
SkillKnowledgeSyncItemRow.lease_owner == owner,
)
.values(
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
await session.commit()
return result.rowcount == 1
async def update_item(
self,
item_id: str,
*,
status: str | None = None,
phase: str | None = None,
progress: int | None = None,
snapshot_id: str | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
if status is not None:
row.status = status
if status == "running" and row.started_at is None:
row.started_at = _now()
row.attempts += 1
if status in {"completed", "partial_failed", "failed", "canceled"}:
row.completed_at = _now()
row.active_dedupe_key = None
elif status == "queued":
row.completed_at = None
if status != "running":
row.lease_owner = None
row.lease_until = None
if phase is not None:
row.phase = phase
if progress is not None:
row.progress = max(0, min(100, progress))
if snapshot_id is not None:
row.snapshot_id = snapshot_id
row.last_error_code = error_code
row.last_error_message = error_message
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def refresh_job(self, job_id: str) -> None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
counts = {str(status): int(count) for status, count in rows}
job.success_count = counts.get("completed", 0)
job.partial_failed_count = counts.get("partial_failed", 0)
job.failed_count = counts.get("failed", 0)
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
if counts.get("running", 0) or counts.get("awaiting_review", 0):
job.status = "running"
job.started_at = job.started_at or _now()
elif active == 0 and sum(counts.values()) >= job.total_count:
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
job.status = "failed"
elif job.failed_count or job.partial_failed_count:
job.status = "partial_failed"
else:
job.status = "completed"
job.completed_at = _now()
await session.commit()
async def cancel_job(self, job_id: str) -> bool:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return False
job.cancel_requested = True
await session.execute(
SkillKnowledgeSyncItemRow.__table__.update()
.where(
SkillKnowledgeSyncItemRow.job_id == job_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
)
.values(
status="canceled",
phase="completed",
completed_at=_now(),
progress=100,
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
)
)
job.status = "canceled"
job.completed_at = _now()
await session.commit()
return True
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
SkillKnowledgeSyncItemRow.id != row.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
return None
row.status = "queued"
row.phase = "scanning"
row.progress = 0
row.completed_at = None
row.active_dedupe_key = row.binding_id
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
stmt = select(SkillKnowledgeSnapshotRow).where(
SkillKnowledgeSnapshotRow.skill_name == skill_name,
SkillKnowledgeSnapshotRow.source_digest == source_digest,
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
SkillKnowledgeSnapshotRow.model_name == model_name,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = SkillKnowledgeSnapshotRow(
id=str(uuid4()),
skill_name=skill_name,
source_digest=source_digest,
extractor_version=extractor_version,
prompt_hash=prompt_hash,
model_name=model_name,
manifest_json=manifest,
ir_json=ir,
counts_json=counts,
status="ready",
created_at=_now(),
)
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
return _dict(row) if row else None
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
rows = [
SkillKnowledgeReviewRow(
id=str(uuid4()),
job_id=job_id,
item_id=item_id,
binding_id=binding_id,
snapshot_id=snapshot_id,
object_type=kind,
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
original_json=obj,
status="pending",
created_at=_now(),
)
for kind, obj in objects
]
async with self._sf() as session:
session.add_all(rows)
await session.commit()
return len(rows)
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeReviewRow)
if job_id:
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
if item_id:
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
if status:
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
final_status = "approved" if action == "approve" else "rejected"
affected_items: set[str] = set()
async with self._sf() as session:
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
for row in rows:
row.status = final_status
row.edited_json = (edits or {}).get(row.id)
row.reason = reason
row.reviewed_by = actor
row.reviewed_at = _now()
affected_items.add(row.item_id)
await session.commit()
return sorted(affected_items)
async def pending_review_count(self, item_id: str) -> int:
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
snapshot = await self.get_snapshot(snapshot_id)
if snapshot is None:
return None
ir = dict(snapshot.get("ir") or {})
reviews = await self.list_reviews(item_id=item_id, status=None)
if not reviews:
return ir
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
for key in ("entities", "relations"):
kind = key[:-1]
accepted = []
for obj in ir.get(key, []):
review = by_key.get((kind, str(obj.get("local_id"))))
if review and review.get("status") == "rejected":
continue
accepted.append((review or {}).get("edited") or obj)
ir[key] = accepted
return ir
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
now = _now()
async with self._sf() as session:
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
if item is None:
raise KeyError(item_id)
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
if binding is None:
raise KeyError(item.binding_id)
if not binding.active:
raise RuntimeError("Binding was detached before staged activation")
await session.execute(
SkillKnowledgeArtifactRow.__table__.update()
.where(
SkillKnowledgeArtifactRow.binding_id == binding.id,
SkillKnowledgeArtifactRow.status == "active",
)
.values(status="retired", updated_at=now)
)
for artifact in artifacts:
session.add(
SkillKnowledgeArtifactRow(
id=str(uuid4()),
binding_id=binding.id,
snapshot_id=snapshot_id,
kind=str(artifact.get("kind") or "wiki_page"),
slug=artifact.get("slug"),
content_text=artifact.get("content"),
remote_id=artifact.get("remote_id"),
business_key=artifact.get("business_key"),
remote_refs_json=artifact.get("remote_refs") or [],
status="active",
metadata_json=artifact.get("metadata") or {},
created_at=now,
updated_at=now,
)
)
# Retire contributions from the old active version only after all staged artifacts are ready.
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
entity_ids: dict[str, str] = {}
for entity in ir.get("entities", []):
normalized = normalize_name(str(entity.get("name") or ""))
if not normalized:
continue
entity_type = str(entity.get("type") or "concept")
stmt = select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == binding.target_type,
SkillKnowledgeEntityRow.target_id == binding.target_id,
SkillKnowledgeEntityRow.entity_type == entity_type,
SkillKnowledgeEntityRow.normalized_name == normalized,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeEntityRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
entity_type=entity_type,
normalized_name=normalized,
display_name=str(entity.get("name") or normalized),
aliases_json=entity.get("aliases") or [],
attributes_json=entity.get("attributes") or {},
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
entity_ids[str(entity.get("local_id"))] = canonical.id
session.add(
SkillKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_entity_id=str(entity.get("local_id")),
evidence_json=entity.get("evidence") or [],
attributes_json=entity.get("attributes") or {},
confidence=float(entity.get("confidence") or 0),
)
)
for relation in ir.get("relations", []):
source_id = entity_ids.get(str(relation.get("source_local_id")))
target_id = entity_ids.get(str(relation.get("target_local_id")))
confidence = float(relation.get("confidence") or 0)
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
continue
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
stmt = select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == binding.target_type,
SkillKnowledgeRelationRow.target_id == binding.target_id,
SkillKnowledgeRelationRow.source_entity_id == source_id,
SkillKnowledgeRelationRow.predicate == predicate,
SkillKnowledgeRelationRow.target_entity_id == target_id,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeRelationRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
source_entity_id=source_id,
predicate=predicate,
target_entity_id=target_id,
description=str(relation.get("description") or ""),
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
session.add(
SkillKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_relation_id=str(relation.get("local_id")),
evidence_json=relation.get("evidence") or [],
confidence=confidence,
)
)
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
binding.current_snapshot_id = snapshot_id
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
binding.synced_digest = binding.source_digest
binding.status = status
binding.last_synced_at = now
binding.last_error_code = None
binding.last_error_message = None
binding.updated_at = now
item.snapshot_id = snapshot_id
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
item.phase = "completed"
item.progress = 100
item.completed_at = now
item.active_dedupe_key = None
item.lease_owner = None
item.lease_until = None
item.updated_at = now
await session.flush()
result = _dict(binding)
await session.commit()
await self.refresh_job(item.job_id)
return result
async def detach_binding(
self,
binding_id: str,
*,
delete_contributions: bool,
cleanup_failed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
if binding is None:
return None
binding.active = False
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
binding.updated_at = _now()
if delete_contributions:
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
await session.flush()
data = _dict(binding)
await session.commit()
return data
async def mark_skill_stale(self, skill_name: str) -> int:
async with self._sf() as session:
result = await session.execute(
SkillKnowledgeBindingRow.__table__.update()
.where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.active.is_(True),
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
)
.values(status="stale", updated_at=_now())
)
await session.commit()
return int(result.rowcount or 0)
def make_skill_knowledge_store(session_factory):
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None