feat: add skill and assistant knowledge bases

offline-backend-20260512/backend/packages/harness/deerflow/assistant_knowledge/__init__.py

@@ -0,0 +1,5 @@
+"""Assistant knowledge package helpers."""
+
+from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
+
+__all__ = ["export_package_from_skill_ir"]

offline-backend-20260512/backend/packages/harness/deerflow/assistant_knowledge/importer.py

@@ -0,0 +1,22 @@
+"""Normalize heterogeneous sources into the assistant import-package shape."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
+    skill = ir.get("skill") or {}
+    return {
+        "source": {
+            "type": "skill",
+            "skill_name": skill.get("name"),
+            "source_digest": skill.get("source_digest"),
+        },
+        "wiki_pages": [ir.get("wiki", {}).get("skill_page")],
+        "documents": ir.get("files") or [],
+        "chunks": [],
+        "entities": ir.get("entities") or [],
+        "relations": ir.get("relations") or [],
+        "graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
+    }

offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py

@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
         repr=False,
         description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
     )
+    graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
     model_config = ConfigDict(extra="ignore")
 
     @field_validator("api_base_url", "web_base_url")
@@ -52,6 +53,23 @@ class WeKnoraConnectionConfig(BaseModel):
         return normalize_service_url(value)
 
 
+class WeKnoraDirectGraphConfig(BaseModel):
+    """Backend-only, opt-in Neo4j transactional HTTP writer settings."""
+
+    direct_write_enabled: bool = False
+    neo4j_http_url: str = ""
+    username: str = ""
+    password: str = Field(default="", repr=False)
+    database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
+    timeout_seconds: float = Field(default=30, gt=0, le=300)
+    model_config = ConfigDict(extra="ignore")
+
+    @field_validator("neo4j_http_url")
+    @classmethod
+    def _validate_url(cls, value: str) -> str:
+        return normalize_service_url(value)
+
+
 class LocalWikiEmbeddingConfig(BaseModel):
     enabled: bool = True
     provider: str = "openai"

offline-backend-20260512/backend/packages/harness/deerflow/config/system_settings.py

@@ -156,6 +156,18 @@ class PromptPrefixSettings(BaseModel):
     model_config = ConfigDict(extra="ignore")
 
 
+class SkillKnowledgeAutoSyncSettings(BaseModel):
+    """Watcher/scheduler policy for bindings that explicitly opt in."""
+
+    watch_enabled: bool = True
+    schedule_enabled: bool = True
+    schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
+    debounce_seconds: int = Field(default=120, ge=10, le=86_400)
+    write_remote_enabled: bool = True
+    review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
+    model_config = ConfigDict(extra="ignore")
+
+
 class AppearanceRouteParam(BaseModel):
     """Runtime query-param descriptor for top-bar shortcut buttons.
 
@@ -501,6 +513,11 @@ class SystemSettings(BaseModel):
     concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
     model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
     llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
+    skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
+    knowledge_base_product_version: Literal["v1", "v2"] = Field(
+        default="v1",
+        description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
+    )
     model_config = ConfigDict(extra="ignore")
 
 

offline-backend-20260512/backend/packages/harness/deerflow/persistence/assistant_knowledge/__init__.py

@@ -0,0 +1,4 @@
+from deerflow.persistence.assistant_knowledge.model import *  # noqa: F401,F403
+from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store  # noqa: F401
+
+__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]

offline-backend-20260512/backend/packages/harness/deerflow/persistence/assistant_knowledge/model.py

@@ -0,0 +1,143 @@
+"""ORM model for the single global DeerFlow assistant knowledge base."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import Float, Integer, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class AssistantKnowledgeBaseRow(Base):
+    __tablename__ = "assistant_knowledge_base"
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    name: Mapped[str] = mapped_column(String(255), default="知识梳理")
+    status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
+    created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class AssistantKnowledgeSourceRow(Base):
+    __tablename__ = "assistant_knowledge_sources"
+    __table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    base_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_type: Mapped[str] = mapped_column(String(32), index=True)
+    source_key: Mapped[str] = mapped_column(String(255))
+    knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    source_name: Mapped[str] = mapped_column(String(255), default="")
+    last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="active")
+    metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class AssistantKnowledgeImportJobRow(Base):
+    __tablename__ = "assistant_knowledge_import_jobs"
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    base_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_id: Mapped[str] = mapped_column(String(36), index=True)
+    trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
+    status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
+    phase: Mapped[str] = mapped_column(String(32), default="exporting")
+    counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
+    created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+
+
+class AssistantKnowledgeWikiPageRow(Base):
+    __tablename__ = "assistant_knowledge_wiki_pages"
+    __table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    base_id: Mapped[str] = mapped_column(String(36), index=True)
+    slug: Mapped[str] = mapped_column(String(512), index=True)
+    title: Mapped[str] = mapped_column(String(512), index=True)
+    page_type: Mapped[str] = mapped_column(String(32), index=True)
+    current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
+    summary: Mapped[str] = mapped_column(String(2048), default="")
+    status: Mapped[str] = mapped_column(String(24), default="active")
+    search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class AssistantKnowledgeWikiRevisionRow(Base):
+    __tablename__ = "assistant_knowledge_wiki_revisions"
+    __table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    page_id: Mapped[str] = mapped_column(String(36), index=True)
+    revision_no: Mapped[int] = mapped_column(Integer)
+    content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
+    summary: Mapped[str] = mapped_column(String(2048), default="")
+    source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    change_type: Mapped[str] = mapped_column(String(32), default="import")
+    status: Mapped[str] = mapped_column(String(24), default="published")
+    created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+
+
+class AssistantKnowledgeEntityRow(Base):
+    __tablename__ = "assistant_knowledge_entities"
+    __table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    base_id: Mapped[str] = mapped_column(String(36), index=True)
+    entity_type: Mapped[str] = mapped_column(String(64), index=True)
+    normalized_name: Mapped[str] = mapped_column(String(512))
+    display_name: Mapped[str] = mapped_column(String(512), index=True)
+    aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="active")
+
+
+class AssistantKnowledgeEntityContributionRow(Base):
+    __tablename__ = "assistant_knowledge_entity_contributions"
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_id: Mapped[str] = mapped_column(String(36), index=True)
+    import_job_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    confidence: Mapped[float] = mapped_column(Float, default=1.0)
+
+
+class AssistantKnowledgeRelationRow(Base):
+    __tablename__ = "assistant_knowledge_relations"
+    __table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    base_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    predicate: Mapped[str] = mapped_column(String(128), index=True)
+    target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    description: Mapped[str] = mapped_column(String(2048), default="")
+    wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="active")
+
+
+class AssistantKnowledgeRelationContributionRow(Base):
+    __tablename__ = "assistant_knowledge_relation_contributions"
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    relation_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_id: Mapped[str] = mapped_column(String(36), index=True)
+    import_job_id: Mapped[str] = mapped_column(String(36), index=True)
+    source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    confidence: Mapped[float] = mapped_column(Float, default=1.0)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/assistant_knowledge/store.py

@@ -0,0 +1,601 @@
+"""Repository for the single global assistant knowledge base."""
+
+from __future__ import annotations
+
+import difflib
+from datetime import UTC, datetime
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import delete, func, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.persistence.assistant_knowledge.model import (
+    AssistantKnowledgeBaseRow,
+    AssistantKnowledgeEntityContributionRow,
+    AssistantKnowledgeEntityRow,
+    AssistantKnowledgeImportJobRow,
+    AssistantKnowledgeRelationContributionRow,
+    AssistantKnowledgeRelationRow,
+    AssistantKnowledgeSourceRow,
+    AssistantKnowledgeWikiPageRow,
+    AssistantKnowledgeWikiRevisionRow,
+)
+from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _dict(row: Any) -> dict[str, Any]:
+    data = row.to_dict()
+    for key, value in tuple(data.items()):
+        if isinstance(value, datetime):
+            data[key] = value.isoformat()
+        elif key.endswith("_json"):
+            data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
+    return data
+
+
+class AssistantKnowledgeRepository:
+    def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+        self._sf = session_factory
+
+    async def get_base(self) -> dict[str, Any] | None:
+        stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
+        async with self._sf() as session:
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            return _dict(row) if row else None
+
+    async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
+        existing = await self.get_base()
+        if existing:
+            return existing
+        row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
+        async with self._sf() as session:
+            session.add(row)
+            await session.flush()
+            data = _dict(row)
+            await session.commit()
+            return data
+
+    async def _upsert_source(
+        self,
+        session: AsyncSession,
+        *,
+        base_id: str,
+        source_type: str,
+        source_key: str,
+        source_name: str,
+        metadata: dict,
+        skill_name: str | None = None,
+        mapping_id: str | None = None,
+        upload_batch_id: str | None = None,
+    ) -> AssistantKnowledgeSourceRow:
+        stmt = select(AssistantKnowledgeSourceRow).where(
+            AssistantKnowledgeSourceRow.base_id == base_id,
+            AssistantKnowledgeSourceRow.source_type == source_type,
+            AssistantKnowledgeSourceRow.source_key == source_key,
+        )
+        row = (await session.execute(stmt)).scalar_one_or_none()
+        if row is None:
+            row = AssistantKnowledgeSourceRow(
+                id=str(uuid4()),
+                base_id=base_id,
+                source_type=source_type,
+                source_key=source_key,
+                source_name=source_name,
+                skill_name=skill_name,
+                knowledge_base_mapping_id=mapping_id,
+                upload_batch_id=upload_batch_id,
+                status="active",
+                metadata_json=metadata,
+                created_at=_now(),
+                updated_at=_now(),
+            )
+            session.add(row)
+            await session.flush()
+        else:
+            row.source_name = source_name
+            row.status = "active"
+            row.metadata_json = metadata
+            row.updated_at = _now()
+        return row
+
+    async def _publish_page(
+        self,
+        session: AsyncSession,
+        *,
+        base_id: str,
+        slug: str,
+        title: str,
+        page_type: str,
+        content: str,
+        summary: str,
+        search_terms: list[str],
+        source_id: str,
+        import_job_id: str,
+        created_by: str | None,
+        change_type: str,
+    ) -> AssistantKnowledgeWikiPageRow:
+        now = _now()
+        page = (
+            await session.execute(
+                select(AssistantKnowledgeWikiPageRow).where(
+                    AssistantKnowledgeWikiPageRow.base_id == base_id,
+                    AssistantKnowledgeWikiPageRow.slug == slug,
+                )
+            )
+        ).scalar_one_or_none()
+        if page is None:
+            page = AssistantKnowledgeWikiPageRow(
+                id=str(uuid4()),
+                base_id=base_id,
+                slug=slug,
+                title=title,
+                page_type=page_type,
+                content_markdown="",
+                summary="",
+                search_text="",
+                status="active",
+                updated_at=now,
+            )
+            session.add(page)
+            await session.flush()
+        revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
+        revision = AssistantKnowledgeWikiRevisionRow(
+            id=str(uuid4()),
+            page_id=page.id,
+            revision_no=revision_no,
+            content_markdown=content,
+            summary=summary,
+            source_id=source_id,
+            import_job_id=import_job_id,
+            change_type=change_type,
+            status="published",
+            created_by=created_by,
+            created_at=now,
+            published_at=now,
+        )
+        session.add(revision)
+        page.title = title
+        page.page_type = page_type
+        page.content_markdown = content
+        page.summary = summary
+        page.search_text = "\n".join([title, summary, content, *search_terms])
+        page.current_revision_id = revision.id
+        page.status = "active"
+        page.updated_at = now
+        return page
+
+    async def import_package(
+        self,
+        *,
+        package: dict[str, Any],
+        source_type: str,
+        source_key: str,
+        source_name: str,
+        trigger: str,
+        created_by: str | None,
+        skill_name: str | None = None,
+        mapping_id: str | None = None,
+        upload_batch_id: str | None = None,
+    ) -> dict[str, Any]:
+        base = await self.initialize(created_by=created_by)
+        now = _now()
+        async with self._sf() as session:
+            source = await self._upsert_source(
+                session,
+                base_id=base["id"],
+                source_type=source_type,
+                source_key=source_key,
+                source_name=source_name,
+                metadata=package.get("source") or {},
+                skill_name=skill_name,
+                mapping_id=mapping_id,
+                upload_batch_id=upload_batch_id,
+            )
+            # A re-import replaces this source's prior contributions while retaining
+            # canonical entities/relations that are still supported by other sources.
+            await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
+            await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
+            job = AssistantKnowledgeImportJobRow(
+                id=str(uuid4()),
+                base_id=base["id"],
+                source_id=source.id,
+                trigger=trigger,
+                status="running",
+                phase="aligning_entities",
+                counts_json={},
+                created_by=created_by,
+                created_at=now,
+            )
+            session.add(job)
+            await session.flush()
+            source.last_import_job_id = job.id
+
+            entity_map: dict[str, str] = {}
+            entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
+            raw_entities = list(package.get("entities") or [])
+            for raw in raw_entities:
+                display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
+                entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
+                normalized = normalize_name(display_name)
+                if not normalized:
+                    continue
+                stmt = select(AssistantKnowledgeEntityRow).where(
+                    AssistantKnowledgeEntityRow.base_id == base["id"],
+                    AssistantKnowledgeEntityRow.entity_type == entity_type,
+                    AssistantKnowledgeEntityRow.normalized_name == normalized,
+                )
+                entity = (await session.execute(stmt)).scalar_one_or_none()
+                if entity is None:
+                    entity = AssistantKnowledgeEntityRow(
+                        id=str(uuid4()),
+                        base_id=base["id"],
+                        entity_type=entity_type,
+                        normalized_name=normalized,
+                        display_name=display_name,
+                        aliases_json=raw.get("aliases") or [],
+                        attributes_json=raw.get("attributes") or {},
+                        status="active",
+                    )
+                    session.add(entity)
+                    await session.flush()
+                raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
+                entity_map[raw_id] = entity.id
+                entity_rows[entity.id] = (entity, raw)
+                session.add(
+                    AssistantKnowledgeEntityContributionRow(
+                        id=str(uuid4()),
+                        entity_id=entity.id,
+                        source_id=source.id,
+                        import_job_id=job.id,
+                        source_wiki_slug=raw.get("source_wiki_slug"),
+                        source_chunk_id=raw.get("source_chunk_id"),
+                        source_skill_name=skill_name,
+                        evidence_json=raw.get("evidence") or [],
+                        confidence=float(raw.get("confidence") or 1.0),
+                    )
+                )
+
+            raw_relations = list(package.get("relations") or [])
+            relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
+            for raw in raw_relations:
+                source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
+                target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
+                confidence = float(raw.get("confidence") or 1.0)
+                if not source_entity_id or not target_entity_id or confidence < 0.75:
+                    continue
+                predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
+                stmt = select(AssistantKnowledgeRelationRow).where(
+                    AssistantKnowledgeRelationRow.base_id == base["id"],
+                    AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
+                    AssistantKnowledgeRelationRow.predicate == predicate,
+                    AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
+                )
+                relation = (await session.execute(stmt)).scalar_one_or_none()
+                if relation is None:
+                    relation = AssistantKnowledgeRelationRow(
+                        id=str(uuid4()),
+                        base_id=base["id"],
+                        source_entity_id=source_entity_id,
+                        predicate=predicate,
+                        target_entity_id=target_entity_id,
+                        description=str(raw.get("description") or ""),
+                        status="active",
+                    )
+                    session.add(relation)
+                    await session.flush()
+                relation_rows[relation.id] = (relation, raw)
+                session.add(
+                    AssistantKnowledgeRelationContributionRow(
+                        id=str(uuid4()),
+                        relation_id=relation.id,
+                        source_id=source.id,
+                        import_job_id=job.id,
+                        source_wiki_slug=raw.get("source_wiki_slug"),
+                        graph_edge_id=raw.get("graph_edge_id"),
+                        evidence_json=raw.get("evidence") or [],
+                        confidence=confidence,
+                    )
+                )
+
+            for entity, raw in entity_rows.values():
+                aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
+                entity.aliases_json = aliases
+                content = "\n".join(
+                    [
+                        f"# {entity.display_name}",
+                        "",
+                        f"- 类型：{entity.entity_type}",
+                        f"- 别名：{'、'.join(aliases) if aliases else '无'}",
+                        f"- 本次来源：{source_name}",
+                        "",
+                        "## 属性",
+                        "",
+                        "```json",
+                        str(entity.attributes_json or {}),
+                        "```",
+                    ]
+                )
+                entity_page = await self._publish_page(
+                    session,
+                    base_id=base["id"],
+                    slug=f"entities/{entity.entity_type}/{entity.id}",
+                    title=entity.display_name,
+                    page_type="entity",
+                    content=content,
+                    summary=f"{entity.entity_type} 实体，来源：{source_name}",
+                    search_terms=aliases,
+                    source_id=source.id,
+                    import_job_id=job.id,
+                    created_by=created_by,
+                    change_type="entity_alignment",
+                )
+                entity.wiki_page_id = entity_page.id
+
+            for relation, raw in relation_rows.values():
+                source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
+                target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
+                source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
+                target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
+                content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源：{source_name}\n- 置信度：{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
+                relation_page = await self._publish_page(
+                    session,
+                    base_id=base["id"],
+                    slug=f"relations/{relation.id}",
+                    title=f"{source_name_display} {relation.predicate} {target_name_display}",
+                    page_type="relation",
+                    content=content,
+                    summary=relation.description,
+                    search_terms=[source_name_display, target_name_display, relation.predicate],
+                    source_id=source.id,
+                    import_job_id=job.id,
+                    created_by=created_by,
+                    change_type="relation_alignment",
+                )
+                relation.wiki_page_id = relation_page.id
+
+            pages = list(package.get("wiki_pages") or [])
+            if not pages and package.get("wiki"):
+                wiki = package["wiki"]
+                page = wiki.get("skill_page") if isinstance(wiki, dict) else None
+                pages = [page] if page else []
+            for raw in pages:
+                if not isinstance(raw, dict):
+                    continue
+                slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
+                title = str(raw.get("title") or source_name)
+                content = str(raw.get("content") or raw.get("content_markdown") or "")
+                summary = str(raw.get("summary") or "")
+                stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
+                page = (await session.execute(stmt)).scalar_one_or_none()
+                if page is None:
+                    page = AssistantKnowledgeWikiPageRow(
+                        id=str(uuid4()),
+                        base_id=base["id"],
+                        slug=slug,
+                        title=title,
+                        page_type=str(raw.get("page_type") or "source"),
+                        content_markdown="",
+                        summary="",
+                        search_text="",
+                        status="active",
+                        updated_at=now,
+                    )
+                    session.add(page)
+                    await session.flush()
+                revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
+                revision = AssistantKnowledgeWikiRevisionRow(
+                    id=str(uuid4()),
+                    page_id=page.id,
+                    revision_no=revision_no,
+                    content_markdown=content,
+                    summary=summary,
+                    source_id=source.id,
+                    import_job_id=job.id,
+                    change_type="skill_sync" if source_type == "skill" else "import",
+                    status="published",
+                    created_by=created_by,
+                    created_at=now,
+                    published_at=now,
+                )
+                session.add(revision)
+                page.title = title
+                page.content_markdown = content
+                page.summary = summary
+                page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
+                page.current_revision_id = revision.id
+                page.updated_at = now
+
+            counts = {
+                "wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
+                "documents": len(package.get("documents") or []),
+                "chunks": len(package.get("chunks") or []),
+                "entities": len(raw_entities),
+                "relations": len(raw_relations),
+                "graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
+                "graph_edges": len((package.get("graph") or {}).get("edges") or []),
+            }
+            job.status = "completed"
+            job.phase = "completed"
+            job.counts_json = counts
+            job.completed_at = now
+            await self._retire_orphans(session, base["id"])
+            await session.flush()
+            result = _dict(job)
+            await session.commit()
+            return result
+
+    async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
+        entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
+        for entity in entities:
+            contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
+            entity.status = "active" if contribution else "deleted"
+        relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
+        for relation in relations:
+            contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
+            relation.status = "active" if contribution else "deleted"
+
+    async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
+        stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
+        if query:
+            needle = f"%{query.strip()}%"
+            stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
+        if page_type:
+            stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
+        stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def get_page(self, slug: str) -> dict[str, Any] | None:
+        stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
+        async with self._sf() as session:
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            return _dict(row) if row else None
+
+    async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
+        async with self._sf() as session:
+            page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
+            if page is None:
+                return []
+            stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
+            return _dict(row) if row else None
+
+    async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
+        left, right = await self.get_revision(left_id), await self.get_revision(right_id)
+        if not left or not right or left["page_id"] != right["page_id"]:
+            return None
+        diff = "\n".join(
+            difflib.unified_diff(
+                str(left["content_markdown"]).splitlines(),
+                str(right["content_markdown"]).splitlines(),
+                fromfile=f"revision-{left['revision_no']}",
+                tofile=f"revision-{right['revision_no']}",
+                lineterm="",
+            )
+        )
+        return {"left": left, "right": right, "diff": diff}
+
+    async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
+        now = _now()
+        async with self._sf() as session:
+            page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
+            source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
+            if page is None or source is None or source.page_id != page.id:
+                return None
+            revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
+            row = AssistantKnowledgeWikiRevisionRow(
+                id=str(uuid4()),
+                page_id=page.id,
+                revision_no=revision_no,
+                content_markdown=source.content_markdown,
+                summary=source.summary,
+                source_id=source.source_id,
+                import_job_id=source.import_job_id,
+                change_type="rollback",
+                status="published",
+                created_by=actor,
+                created_at=now,
+                published_at=now,
+            )
+            session.add(row)
+            page.current_revision_id = row.id
+            page.content_markdown = row.content_markdown
+            page.summary = row.summary
+            page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
+            page.updated_at = now
+            await session.flush()
+            data = _dict(row)
+            await session.commit()
+            return data
+
+    async def list_sources(self) -> list[dict[str, Any]]:
+        async with self._sf() as session:
+            stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def get_source(self, source_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(AssistantKnowledgeSourceRow, source_id)
+            return _dict(row) if row and row.status != "deleted" else None
+
+    async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(AssistantKnowledgeEntityRow, entity_id)
+            if row is None or row.status != "active":
+                return None
+            contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
+            result = _dict(row)
+            result["contributions"] = [_dict(value) for value in contributions]
+            return result
+
+    async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(AssistantKnowledgeRelationRow, relation_id)
+            if row is None or row.status != "active":
+                return None
+            contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
+            result = _dict(row)
+            result["contributions"] = [_dict(value) for value in contributions]
+            return result
+
+    async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
+        async with self._sf() as session:
+            stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def get_job(self, job_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(AssistantKnowledgeImportJobRow, job_id)
+            return _dict(row) if row else None
+
+    async def delete_source(self, source_id: str) -> bool:
+        async with self._sf() as session:
+            source = await session.get(AssistantKnowledgeSourceRow, source_id)
+            if source is None:
+                return False
+            source.status = "deleted"
+            await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
+            await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
+            pages = (
+                await session.execute(
+                    select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
+                )
+            ).scalars()
+            for page in pages:
+                fallback = (
+                    await session.execute(
+                        select(AssistantKnowledgeWikiRevisionRow)
+                        .where(
+                            AssistantKnowledgeWikiRevisionRow.page_id == page.id,
+                            AssistantKnowledgeWikiRevisionRow.source_id != source_id,
+                            AssistantKnowledgeWikiRevisionRow.status == "published",
+                        )
+                        .order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
+                        .limit(1)
+                    )
+                ).scalar_one_or_none()
+                if fallback is None:
+                    page.status = "deleted"
+                    page.current_revision_id = None
+                else:
+                    page.current_revision_id = fallback.id
+                    page.content_markdown = fallback.content_markdown
+                    page.summary = fallback.summary
+                    page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
+            await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
+            await self._retire_orphans(session, source.base_id)
+            await session.commit()
+            return True
+
+
+def make_assistant_knowledge_store(session_factory):
+    return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None

offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260901_01_skill_and_assistant_knowledge.py

@@ -0,0 +1,73 @@
+"""Add skill distillation and assistant knowledge tables.
+
+Revision ID: 20260901_01
+Revises: 20260825_01
+Create Date: 2026-09-01
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+from deerflow.persistence.assistant_knowledge.model import (
+    AssistantKnowledgeBaseRow,
+    AssistantKnowledgeEntityContributionRow,
+    AssistantKnowledgeEntityRow,
+    AssistantKnowledgeImportJobRow,
+    AssistantKnowledgeRelationContributionRow,
+    AssistantKnowledgeRelationRow,
+    AssistantKnowledgeSourceRow,
+    AssistantKnowledgeWikiPageRow,
+    AssistantKnowledgeWikiRevisionRow,
+)
+from deerflow.persistence.skill_knowledge.model import (
+    SkillKnowledgeArtifactRow,
+    SkillKnowledgeBindingRow,
+    SkillKnowledgeEntityContributionRow,
+    SkillKnowledgeEntityRow,
+    SkillKnowledgeRelationContributionRow,
+    SkillKnowledgeRelationRow,
+    SkillKnowledgeReviewRow,
+    SkillKnowledgeSnapshotRow,
+    SkillKnowledgeSyncItemRow,
+    SkillKnowledgeSyncJobRow,
+)
+
+revision = "20260901_01"
+down_revision = "20260825_01"
+branch_labels = None
+depends_on = None
+
+TABLES = [
+    SkillKnowledgeBindingRow,
+    SkillKnowledgeSnapshotRow,
+    SkillKnowledgeSyncJobRow,
+    SkillKnowledgeSyncItemRow,
+    SkillKnowledgeArtifactRow,
+    SkillKnowledgeReviewRow,
+    SkillKnowledgeEntityRow,
+    SkillKnowledgeEntityContributionRow,
+    SkillKnowledgeRelationRow,
+    SkillKnowledgeRelationContributionRow,
+    AssistantKnowledgeBaseRow,
+    AssistantKnowledgeSourceRow,
+    AssistantKnowledgeImportJobRow,
+    AssistantKnowledgeWikiPageRow,
+    AssistantKnowledgeWikiRevisionRow,
+    AssistantKnowledgeEntityRow,
+    AssistantKnowledgeEntityContributionRow,
+    AssistantKnowledgeRelationRow,
+    AssistantKnowledgeRelationContributionRow,
+]
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    for model in TABLES:
+        model.__table__.create(bind, checkfirst=True)
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    for model in reversed(TABLES):
+        model.__table__.drop(bind, checkfirst=True)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py

@@ -35,6 +35,17 @@ from deerflow.persistence.agents.model import (
     AgentRow,
     AgentSkillRow,
 )
+from deerflow.persistence.assistant_knowledge.model import (
+    AssistantKnowledgeBaseRow,
+    AssistantKnowledgeEntityContributionRow,
+    AssistantKnowledgeEntityRow,
+    AssistantKnowledgeImportJobRow,
+    AssistantKnowledgeRelationContributionRow,
+    AssistantKnowledgeRelationRow,
+    AssistantKnowledgeSourceRow,
+    AssistantKnowledgeWikiPageRow,
+    AssistantKnowledgeWikiRevisionRow,
+)
 from deerflow.persistence.business_mapping.model import BusinessMappingRow
 from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
 from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -86,6 +97,18 @@ from deerflow.persistence.scheduled_tasks.model import (
     SchedulerThreadRow,
 )
 from deerflow.persistence.sensitive_words.model import SensitiveWordRow
+from deerflow.persistence.skill_knowledge.model import (
+    SkillKnowledgeArtifactRow,
+    SkillKnowledgeBindingRow,
+    SkillKnowledgeEntityContributionRow,
+    SkillKnowledgeEntityRow,
+    SkillKnowledgeRelationContributionRow,
+    SkillKnowledgeRelationRow,
+    SkillKnowledgeReviewRow,
+    SkillKnowledgeSnapshotRow,
+    SkillKnowledgeSyncItemRow,
+    SkillKnowledgeSyncJobRow,
+)
 from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
 from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
 from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -107,6 +130,15 @@ __all__ = [
     "AgentSkillRow",
     "AdminLeaderboardDailyStatRow",
     "BusinessMappingRow",
+    "AssistantKnowledgeBaseRow",
+    "AssistantKnowledgeSourceRow",
+    "AssistantKnowledgeImportJobRow",
+    "AssistantKnowledgeWikiPageRow",
+    "AssistantKnowledgeWikiRevisionRow",
+    "AssistantKnowledgeEntityRow",
+    "AssistantKnowledgeEntityContributionRow",
+    "AssistantKnowledgeRelationRow",
+    "AssistantKnowledgeRelationContributionRow",
     "CompactMenuOverrideRow",
     "ConcurrencySampleRow",
     "DashboardSessionRow",
@@ -164,6 +196,16 @@ __all__ = [
     "SkillRow",
     "SkillDisplayAdapterRow",
     "SkillFavoriteRow",
+    "SkillKnowledgeArtifactRow",
+    "SkillKnowledgeBindingRow",
+    "SkillKnowledgeEntityContributionRow",
+    "SkillKnowledgeEntityRow",
+    "SkillKnowledgeRelationContributionRow",
+    "SkillKnowledgeRelationRow",
+    "SkillKnowledgeReviewRow",
+    "SkillKnowledgeSnapshotRow",
+    "SkillKnowledgeSyncItemRow",
+    "SkillKnowledgeSyncJobRow",
     "TagAssignmentRow",
     "TagRow",
     "TaskButtonRow",

offline-backend-20260512/backend/packages/harness/deerflow/persistence/skill_knowledge/__init__.py

@@ -0,0 +1,15 @@
+from deerflow.persistence.skill_knowledge.model import (  # noqa: F401
+    SkillKnowledgeArtifactRow,
+    SkillKnowledgeBindingRow,
+    SkillKnowledgeEntityContributionRow,
+    SkillKnowledgeEntityRow,
+    SkillKnowledgeRelationContributionRow,
+    SkillKnowledgeRelationRow,
+    SkillKnowledgeReviewRow,
+    SkillKnowledgeSnapshotRow,
+    SkillKnowledgeSyncItemRow,
+    SkillKnowledgeSyncJobRow,
+)
+from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store  # noqa: F401
+
+__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]

offline-backend-20260512/backend/packages/harness/deerflow/persistence/skill_knowledge/model.py

@@ -0,0 +1,203 @@
+"""ORM rows for skill-to-knowledge distillation and review."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class SkillKnowledgeBindingRow(Base):
+    __tablename__ = "skill_knowledge_bindings"
+    __table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    skill_name: Mapped[str] = mapped_column(String(255), index=True)
+    target_type: Mapped[str] = mapped_column(String(32), index=True)
+    target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
+    target_id: Mapped[str] = mapped_column(String(128), index=True)
+    target_name: Mapped[str] = mapped_column(String(255), default="")
+    remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
+    active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
+    auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
+    current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
+    synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
+    last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
+    created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class SkillKnowledgeSnapshotRow(Base):
+    __tablename__ = "skill_knowledge_snapshots"
+    __table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    skill_name: Mapped[str] = mapped_column(String(255), index=True)
+    source_digest: Mapped[str] = mapped_column(String(80), index=True)
+    extractor_version: Mapped[str] = mapped_column(String(64))
+    prompt_hash: Mapped[str] = mapped_column(String(80))
+    model_name: Mapped[str] = mapped_column(String(128))
+    manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="ready")
+    counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+
+
+class SkillKnowledgeSyncJobRow(Base):
+    __tablename__ = "skill_knowledge_sync_jobs"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    trigger: Mapped[str] = mapped_column(String(32), default="manual")
+    target_count: Mapped[int] = mapped_column(Integer, default=0)
+    targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
+    total_count: Mapped[int] = mapped_column(Integer, default=0)
+    success_count: Mapped[int] = mapped_column(Integer, default=0)
+    partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
+    failed_count: Mapped[int] = mapped_column(Integer, default=0)
+    cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
+    review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
+    created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+
+
+class SkillKnowledgeSyncItemRow(Base):
+    __tablename__ = "skill_knowledge_sync_items"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    job_id: Mapped[str] = mapped_column(String(36), index=True)
+    binding_id: Mapped[str] = mapped_column(String(36), index=True)
+    target_type: Mapped[str] = mapped_column(String(32))
+    target_mode: Mapped[str] = mapped_column(String(32))
+    target_id: Mapped[str] = mapped_column(String(128))
+    skill_name: Mapped[str] = mapped_column(String(255), index=True)
+    snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
+    phase: Mapped[str] = mapped_column(String(32), default="scanning")
+    progress: Mapped[int] = mapped_column(Integer, default=0)
+    attempts: Mapped[int] = mapped_column(Integer, default=0)
+    active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
+    lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
+    lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
+    last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+    started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+
+
+class SkillKnowledgeArtifactRow(Base):
+    __tablename__ = "skill_knowledge_artifacts"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    binding_id: Mapped[str] = mapped_column(String(36), index=True)
+    snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
+    kind: Mapped[str] = mapped_column(String(48), index=True)
+    slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
+    remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
+    metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class SkillKnowledgeReviewRow(Base):
+    __tablename__ = "skill_knowledge_reviews"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    job_id: Mapped[str] = mapped_column(String(36), index=True)
+    item_id: Mapped[str] = mapped_column(String(36), index=True)
+    binding_id: Mapped[str] = mapped_column(String(36), index=True)
+    snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
+    object_type: Mapped[str] = mapped_column(String(32), index=True)
+    object_key: Mapped[str] = mapped_column(String(128))
+    original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
+    reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+
+
+class SkillKnowledgeEntityRow(Base):
+    __tablename__ = "skill_knowledge_entities"
+    __table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    target_type: Mapped[str] = mapped_column(String(32), index=True)
+    target_id: Mapped[str] = mapped_column(String(128), index=True)
+    entity_type: Mapped[str] = mapped_column(String(64), index=True)
+    normalized_name: Mapped[str] = mapped_column(String(512))
+    display_name: Mapped[str] = mapped_column(String(512))
+    aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    status: Mapped[str] = mapped_column(String(24), default="active")
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class SkillKnowledgeEntityContributionRow(Base):
+    __tablename__ = "skill_knowledge_entity_contributions"
+    __table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    binding_id: Mapped[str] = mapped_column(String(36), index=True)
+    snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
+    local_entity_id: Mapped[str] = mapped_column(String(128))
+    evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
+    confidence: Mapped[float] = mapped_column(Float, default=1.0)
+
+
+class SkillKnowledgeRelationRow(Base):
+    __tablename__ = "skill_knowledge_relations"
+    __table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    target_type: Mapped[str] = mapped_column(String(32), index=True)
+    target_id: Mapped[str] = mapped_column(String(128), index=True)
+    source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    predicate: Mapped[str] = mapped_column(String(128), index=True)
+    target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
+    description: Mapped[str] = mapped_column(String(2048), default="")
+    status: Mapped[str] = mapped_column(String(24), default="active")
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
+
+
+class SkillKnowledgeRelationContributionRow(Base):
+    __tablename__ = "skill_knowledge_relation_contributions"
+    __table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    relation_id: Mapped[str] = mapped_column(String(36), index=True)
+    binding_id: Mapped[str] = mapped_column(String(36), index=True)
+    snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
+    local_relation_id: Mapped[str] = mapped_column(String(128))
+    evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
+    confidence: Mapped[float] = mapped_column(Float, default=1.0)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/skill_knowledge/store.py

@@ -0,0 +1,758 @@
+"""SQL repository for skill knowledge jobs, snapshots and contributions."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import delete, func, or_, select, update
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.persistence.skill_knowledge.model import (
+    SkillKnowledgeArtifactRow,
+    SkillKnowledgeBindingRow,
+    SkillKnowledgeEntityContributionRow,
+    SkillKnowledgeEntityRow,
+    SkillKnowledgeRelationContributionRow,
+    SkillKnowledgeRelationRow,
+    SkillKnowledgeReviewRow,
+    SkillKnowledgeSnapshotRow,
+    SkillKnowledgeSyncItemRow,
+    SkillKnowledgeSyncJobRow,
+)
+from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _dict(row: Any) -> dict[str, Any]:
+    list_json_fields = {
+        "aliases_json",
+        "evidence_json",
+        "manifest_json",
+        "refs_json",
+        "targets_json",
+    }
+    data = row.to_dict()
+    for key, value in tuple(data.items()):
+        if isinstance(value, datetime):
+            data[key] = value.isoformat()
+        elif key.endswith("_json"):
+            data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
+    return data
+
+
+class SkillKnowledgeRepository:
+    def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+        self._sf = session_factory
+
+    async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
+        entities = (
+            await session.execute(
+                select(SkillKnowledgeEntityRow).where(
+                    SkillKnowledgeEntityRow.target_type == target_type,
+                    SkillKnowledgeEntityRow.target_id == target_id,
+                )
+            )
+        ).scalars()
+        for entity in entities:
+            contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
+            entity.status = "active" if contribution else "deleted"
+        relations = (
+            await session.execute(
+                select(SkillKnowledgeRelationRow).where(
+                    SkillKnowledgeRelationRow.target_type == target_type,
+                    SkillKnowledgeRelationRow.target_id == target_id,
+                )
+            )
+        ).scalars()
+        for relation in relations:
+            contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
+            relation.status = "active" if contribution else "deleted"
+
+    async def create_job(
+        self,
+        *,
+        skill_names: list[str],
+        targets: list[dict[str, Any]],
+        created_by: str | None,
+        force: bool = False,
+        trigger: str = "manual",
+        review_mode: str = "auto_high_confidence",
+    ) -> dict[str, Any]:
+        names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
+        normalized_targets: list[dict[str, Any]] = []
+        seen_targets: set[tuple[str, str]] = set()
+        for target in targets:
+            target_type = str(target.get("target_type") or "").strip()
+            target_id = str(target.get("target_id") or "").strip()
+            if not target_type or not target_id or (target_type, target_id) in seen_targets:
+                continue
+            seen_targets.add((target_type, target_id))
+            normalized_targets.append(
+                {
+                    "target_type": target_type,
+                    "target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
+                    "target_id": target_id,
+                    "target_name": str(target.get("target_name") or target_id),
+                    "remote_business_key": target.get("remote_business_key"),
+                    "readonly": bool(target.get("readonly", False)),
+                }
+            )
+        job_id = str(uuid4())
+        now = _now()
+        async with self._sf() as session:
+            job = SkillKnowledgeSyncJobRow(
+                id=job_id,
+                trigger=trigger,
+                target_count=len(normalized_targets),
+                targets_json=normalized_targets,
+                status="queued",
+                total_count=0,
+                review_mode=review_mode,
+                created_by=created_by,
+                created_at=now,
+            )
+            session.add(job)
+            for skill_name in names:
+                for target in normalized_targets:
+                    stmt = select(SkillKnowledgeBindingRow).where(
+                        SkillKnowledgeBindingRow.skill_name == skill_name,
+                        SkillKnowledgeBindingRow.target_type == target["target_type"],
+                        SkillKnowledgeBindingRow.target_id == target["target_id"],
+                    )
+                    binding = (await session.execute(stmt)).scalar_one_or_none()
+                    if binding is None:
+                        binding = SkillKnowledgeBindingRow(
+                            id=str(uuid4()),
+                            skill_name=skill_name,
+                            target_type=target["target_type"],
+                            target_mode=target["target_mode"],
+                            target_id=target["target_id"],
+                            target_name=target["target_name"],
+                            remote_business_key=target.get("remote_business_key"),
+                            status="queued",
+                            created_by=created_by,
+                            created_at=now,
+                            updated_at=now,
+                        )
+                        session.add(binding)
+                    else:
+                        active_item = (
+                            await session.execute(
+                                select(SkillKnowledgeSyncItemRow.id)
+                                .where(
+                                    SkillKnowledgeSyncItemRow.binding_id == binding.id,
+                                    SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
+                                )
+                                .limit(1)
+                            )
+                        ).scalar_one_or_none()
+                        if active_item is not None:
+                            continue
+                        binding.target_mode = target["target_mode"]
+                        binding.target_name = target["target_name"]
+                        binding.remote_business_key = target.get("remote_business_key")
+                        binding.status = "queued"
+                        binding.active = True
+                        binding.updated_at = now
+                    item = SkillKnowledgeSyncItemRow(
+                        id=str(uuid4()),
+                        job_id=job_id,
+                        binding_id=binding.id,
+                        target_type=target["target_type"],
+                        target_mode=target["target_mode"],
+                        target_id=target["target_id"],
+                        skill_name=skill_name,
+                        status="queued",
+                        phase="scanning",
+                        progress=0,
+                        active_dedupe_key=binding.id,
+                        created_at=now,
+                        updated_at=now,
+                    )
+                    session.add(item)
+                    binding.last_job_id = job_id
+                    binding.last_item_id = item.id
+                    job.total_count += 1
+            if job.total_count == 0:
+                job.status = "completed"
+                job.completed_at = now
+            await session.flush()
+            result = _dict(job)
+            result["force"] = force
+            await session.commit()
+        return result
+
+    async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
+        stmt = select(SkillKnowledgeBindingRow)
+        if skill_names:
+            stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
+        if target_id:
+            stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
+        if not include_detached:
+            stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
+        stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(SkillKnowledgeBindingRow, binding_id)
+            return _dict(row) if row else None
+
+    async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
+        stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
+        if active_only:
+            stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def has_active_item(self, binding_id: str) -> bool:
+        async with self._sf() as session:
+            value = (
+                await session.execute(
+                    select(SkillKnowledgeSyncItemRow.id)
+                    .where(
+                        SkillKnowledgeSyncItemRow.binding_id == binding_id,
+                        SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
+                    )
+                    .limit(1)
+                )
+            ).scalar_one_or_none()
+            return value is not None
+
+    async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(SkillKnowledgeBindingRow, binding_id)
+            if row is None or not row.active:
+                return None
+            row.auto_sync_enabled = enabled
+            row.updated_at = _now()
+            await session.flush()
+            data = _dict(row)
+            await session.commit()
+            return data
+
+    async def get_job(self, job_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            job = await session.get(SkillKnowledgeSyncJobRow, job_id)
+            if job is None:
+                return None
+            items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
+            result = _dict(job)
+            result["items"] = [_dict(item) for item in items]
+            return result
+
+    async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
+        stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
+        now = _now()
+        stmt = (
+            select(SkillKnowledgeSyncItemRow)
+            .where(
+                or_(
+                    SkillKnowledgeSyncItemRow.status == "queued",
+                    ((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
+                )
+            )
+            .order_by(SkillKnowledgeSyncItemRow.created_at)
+            .limit(limit)
+        )
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def claim_item(
+        self,
+        item_id: str,
+        *,
+        owner: str,
+        lease_seconds: int = 900,
+    ) -> dict[str, Any] | None:
+        """Atomically claim a queued item, or reclaim one whose worker lease expired."""
+        now = _now()
+        stmt = (
+            update(SkillKnowledgeSyncItemRow)
+            .where(
+                SkillKnowledgeSyncItemRow.id == item_id,
+                or_(
+                    SkillKnowledgeSyncItemRow.status == "queued",
+                    ((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
+                ),
+            )
+            .values(
+                status="running",
+                started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
+                attempts=SkillKnowledgeSyncItemRow.attempts + 1,
+                lease_owner=owner,
+                lease_until=now + timedelta(seconds=lease_seconds),
+                updated_at=now,
+            )
+        )
+        async with self._sf() as session:
+            result = await session.execute(stmt)
+            if result.rowcount != 1:
+                await session.rollback()
+                return None
+            row = await session.get(SkillKnowledgeSyncItemRow, item_id)
+            data = _dict(row)
+            await session.commit()
+        await self.refresh_job(data["job_id"])
+        return data
+
+    async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
+        now = _now()
+        async with self._sf() as session:
+            result = await session.execute(
+                update(SkillKnowledgeSyncItemRow)
+                .where(
+                    SkillKnowledgeSyncItemRow.id == item_id,
+                    SkillKnowledgeSyncItemRow.status == "running",
+                    SkillKnowledgeSyncItemRow.lease_owner == owner,
+                )
+                .values(
+                    lease_until=now + timedelta(seconds=lease_seconds),
+                    updated_at=now,
+                )
+            )
+            await session.commit()
+            return result.rowcount == 1
+
+    async def update_item(
+        self,
+        item_id: str,
+        *,
+        status: str | None = None,
+        phase: str | None = None,
+        progress: int | None = None,
+        snapshot_id: str | None = None,
+        error_code: str | None = None,
+        error_message: str | None = None,
+    ) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(SkillKnowledgeSyncItemRow, item_id)
+            if row is None:
+                return None
+            if status is not None:
+                row.status = status
+                if status == "running" and row.started_at is None:
+                    row.started_at = _now()
+                    row.attempts += 1
+                if status in {"completed", "partial_failed", "failed", "canceled"}:
+                    row.completed_at = _now()
+                    row.active_dedupe_key = None
+                elif status == "queued":
+                    row.completed_at = None
+                if status != "running":
+                    row.lease_owner = None
+                    row.lease_until = None
+            if phase is not None:
+                row.phase = phase
+            if progress is not None:
+                row.progress = max(0, min(100, progress))
+            if snapshot_id is not None:
+                row.snapshot_id = snapshot_id
+            row.last_error_code = error_code
+            row.last_error_message = error_message
+            row.updated_at = _now()
+            await session.flush()
+            data = _dict(row)
+            await session.commit()
+        await self.refresh_job(data["job_id"])
+        return data
+
+    async def refresh_job(self, job_id: str) -> None:
+        async with self._sf() as session:
+            job = await session.get(SkillKnowledgeSyncJobRow, job_id)
+            if job is None:
+                return
+            rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
+            counts = {str(status): int(count) for status, count in rows}
+            job.success_count = counts.get("completed", 0)
+            job.partial_failed_count = counts.get("partial_failed", 0)
+            job.failed_count = counts.get("failed", 0)
+            active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
+            if counts.get("running", 0) or counts.get("awaiting_review", 0):
+                job.status = "running"
+                job.started_at = job.started_at or _now()
+            elif active == 0 and sum(counts.values()) >= job.total_count:
+                if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
+                    job.status = "failed"
+                elif job.failed_count or job.partial_failed_count:
+                    job.status = "partial_failed"
+                else:
+                    job.status = "completed"
+                job.completed_at = _now()
+            await session.commit()
+
+    async def cancel_job(self, job_id: str) -> bool:
+        async with self._sf() as session:
+            job = await session.get(SkillKnowledgeSyncJobRow, job_id)
+            if job is None:
+                return False
+            job.cancel_requested = True
+            await session.execute(
+                SkillKnowledgeSyncItemRow.__table__.update()
+                .where(
+                    SkillKnowledgeSyncItemRow.job_id == job_id,
+                    SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
+                )
+                .values(
+                    status="canceled",
+                    phase="completed",
+                    completed_at=_now(),
+                    progress=100,
+                    active_dedupe_key=None,
+                    lease_owner=None,
+                    lease_until=None,
+                )
+            )
+            job.status = "canceled"
+            job.completed_at = _now()
+            await session.commit()
+            return True
+
+    async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(SkillKnowledgeSyncItemRow, item_id)
+            if row is None:
+                return None
+            active_item = (
+                await session.execute(
+                    select(SkillKnowledgeSyncItemRow.id)
+                    .where(
+                        SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
+                        SkillKnowledgeSyncItemRow.id != row.id,
+                        SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
+                    )
+                    .limit(1)
+                )
+            ).scalar_one_or_none()
+            if active_item is not None:
+                return None
+            row.status = "queued"
+            row.phase = "scanning"
+            row.progress = 0
+            row.completed_at = None
+            row.active_dedupe_key = row.binding_id
+            row.updated_at = _now()
+            await session.flush()
+            data = _dict(row)
+            await session.commit()
+        await self.refresh_job(data["job_id"])
+        return data
+
+    async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
+        stmt = select(SkillKnowledgeSnapshotRow).where(
+            SkillKnowledgeSnapshotRow.skill_name == skill_name,
+            SkillKnowledgeSnapshotRow.source_digest == source_digest,
+            SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
+            SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
+            SkillKnowledgeSnapshotRow.model_name == model_name,
+        )
+        async with self._sf() as session:
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            if row is None:
+                row = SkillKnowledgeSnapshotRow(
+                    id=str(uuid4()),
+                    skill_name=skill_name,
+                    source_digest=source_digest,
+                    extractor_version=extractor_version,
+                    prompt_hash=prompt_hash,
+                    model_name=model_name,
+                    manifest_json=manifest,
+                    ir_json=ir,
+                    counts_json=counts,
+                    status="ready",
+                    created_at=_now(),
+                )
+                session.add(row)
+                await session.flush()
+            data = _dict(row)
+            await session.commit()
+            return data
+
+    async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
+            return _dict(row) if row else None
+
+    async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
+        rows = [
+            SkillKnowledgeReviewRow(
+                id=str(uuid4()),
+                job_id=job_id,
+                item_id=item_id,
+                binding_id=binding_id,
+                snapshot_id=snapshot_id,
+                object_type=kind,
+                object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
+                original_json=obj,
+                status="pending",
+                created_at=_now(),
+            )
+            for kind, obj in objects
+        ]
+        async with self._sf() as session:
+            session.add_all(rows)
+            await session.commit()
+        return len(rows)
+
+    async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
+        stmt = select(SkillKnowledgeReviewRow)
+        if job_id:
+            stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
+        if item_id:
+            stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
+        if status:
+            stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
+        stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
+        final_status = "approved" if action == "approve" else "rejected"
+        affected_items: set[str] = set()
+        async with self._sf() as session:
+            rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
+            for row in rows:
+                row.status = final_status
+                row.edited_json = (edits or {}).get(row.id)
+                row.reason = reason
+                row.reviewed_by = actor
+                row.reviewed_at = _now()
+                affected_items.add(row.item_id)
+            await session.commit()
+        return sorted(affected_items)
+
+    async def pending_review_count(self, item_id: str) -> int:
+        stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
+        async with self._sf() as session:
+            return int((await session.execute(stmt)).scalar_one())
+
+    async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
+        snapshot = await self.get_snapshot(snapshot_id)
+        if snapshot is None:
+            return None
+        ir = dict(snapshot.get("ir") or {})
+        reviews = await self.list_reviews(item_id=item_id, status=None)
+        if not reviews:
+            return ir
+        by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
+        for key in ("entities", "relations"):
+            kind = key[:-1]
+            accepted = []
+            for obj in ir.get(key, []):
+                review = by_key.get((kind, str(obj.get("local_id"))))
+                if review and review.get("status") == "rejected":
+                    continue
+                accepted.append((review or {}).get("edited") or obj)
+            ir[key] = accepted
+        return ir
+
+    async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
+        now = _now()
+        async with self._sf() as session:
+            item = await session.get(SkillKnowledgeSyncItemRow, item_id)
+            if item is None:
+                raise KeyError(item_id)
+            binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
+            if binding is None:
+                raise KeyError(item.binding_id)
+            if not binding.active:
+                raise RuntimeError("Binding was detached before staged activation")
+            await session.execute(
+                SkillKnowledgeArtifactRow.__table__.update()
+                .where(
+                    SkillKnowledgeArtifactRow.binding_id == binding.id,
+                    SkillKnowledgeArtifactRow.status == "active",
+                )
+                .values(status="retired", updated_at=now)
+            )
+            for artifact in artifacts:
+                session.add(
+                    SkillKnowledgeArtifactRow(
+                        id=str(uuid4()),
+                        binding_id=binding.id,
+                        snapshot_id=snapshot_id,
+                        kind=str(artifact.get("kind") or "wiki_page"),
+                        slug=artifact.get("slug"),
+                        content_text=artifact.get("content"),
+                        remote_id=artifact.get("remote_id"),
+                        business_key=artifact.get("business_key"),
+                        remote_refs_json=artifact.get("remote_refs") or [],
+                        status="active",
+                        metadata_json=artifact.get("metadata") or {},
+                        created_at=now,
+                        updated_at=now,
+                    )
+                )
+            # Retire contributions from the old active version only after all staged artifacts are ready.
+            await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
+            await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
+            entity_ids: dict[str, str] = {}
+            for entity in ir.get("entities", []):
+                normalized = normalize_name(str(entity.get("name") or ""))
+                if not normalized:
+                    continue
+                entity_type = str(entity.get("type") or "concept")
+                stmt = select(SkillKnowledgeEntityRow).where(
+                    SkillKnowledgeEntityRow.target_type == binding.target_type,
+                    SkillKnowledgeEntityRow.target_id == binding.target_id,
+                    SkillKnowledgeEntityRow.entity_type == entity_type,
+                    SkillKnowledgeEntityRow.normalized_name == normalized,
+                )
+                canonical = (await session.execute(stmt)).scalar_one_or_none()
+                if canonical is None:
+                    canonical = SkillKnowledgeEntityRow(
+                        id=str(uuid4()),
+                        target_type=binding.target_type,
+                        target_id=binding.target_id,
+                        entity_type=entity_type,
+                        normalized_name=normalized,
+                        display_name=str(entity.get("name") or normalized),
+                        aliases_json=entity.get("aliases") or [],
+                        attributes_json=entity.get("attributes") or {},
+                        status="active",
+                        created_at=now,
+                        updated_at=now,
+                    )
+                    session.add(canonical)
+                    await session.flush()
+                else:
+                    canonical.status = "active"
+                entity_ids[str(entity.get("local_id"))] = canonical.id
+                session.add(
+                    SkillKnowledgeEntityContributionRow(
+                        id=str(uuid4()),
+                        entity_id=canonical.id,
+                        binding_id=binding.id,
+                        snapshot_id=snapshot_id,
+                        local_entity_id=str(entity.get("local_id")),
+                        evidence_json=entity.get("evidence") or [],
+                        attributes_json=entity.get("attributes") or {},
+                        confidence=float(entity.get("confidence") or 0),
+                    )
+                )
+            for relation in ir.get("relations", []):
+                source_id = entity_ids.get(str(relation.get("source_local_id")))
+                target_id = entity_ids.get(str(relation.get("target_local_id")))
+                confidence = float(relation.get("confidence") or 0)
+                if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
+                    continue
+                predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
+                stmt = select(SkillKnowledgeRelationRow).where(
+                    SkillKnowledgeRelationRow.target_type == binding.target_type,
+                    SkillKnowledgeRelationRow.target_id == binding.target_id,
+                    SkillKnowledgeRelationRow.source_entity_id == source_id,
+                    SkillKnowledgeRelationRow.predicate == predicate,
+                    SkillKnowledgeRelationRow.target_entity_id == target_id,
+                )
+                canonical = (await session.execute(stmt)).scalar_one_or_none()
+                if canonical is None:
+                    canonical = SkillKnowledgeRelationRow(
+                        id=str(uuid4()),
+                        target_type=binding.target_type,
+                        target_id=binding.target_id,
+                        source_entity_id=source_id,
+                        predicate=predicate,
+                        target_entity_id=target_id,
+                        description=str(relation.get("description") or ""),
+                        status="active",
+                        created_at=now,
+                        updated_at=now,
+                    )
+                    session.add(canonical)
+                    await session.flush()
+                else:
+                    canonical.status = "active"
+                session.add(
+                    SkillKnowledgeRelationContributionRow(
+                        id=str(uuid4()),
+                        relation_id=canonical.id,
+                        binding_id=binding.id,
+                        snapshot_id=snapshot_id,
+                        local_relation_id=str(relation.get("local_id")),
+                        evidence_json=relation.get("evidence") or [],
+                        confidence=confidence,
+                    )
+                )
+            await self._retire_target_orphans(
+                session,
+                target_type=binding.target_type,
+                target_id=binding.target_id,
+            )
+            binding.current_snapshot_id = snapshot_id
+            binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
+            binding.synced_digest = binding.source_digest
+            binding.status = status
+            binding.last_synced_at = now
+            binding.last_error_code = None
+            binding.last_error_message = None
+            binding.updated_at = now
+            item.snapshot_id = snapshot_id
+            item.status = "completed" if status in {"synced", "pending_remote_write"} else status
+            item.phase = "completed"
+            item.progress = 100
+            item.completed_at = now
+            item.active_dedupe_key = None
+            item.lease_owner = None
+            item.lease_until = None
+            item.updated_at = now
+            await session.flush()
+            result = _dict(binding)
+            await session.commit()
+        await self.refresh_job(item.job_id)
+        return result
+
+    async def detach_binding(
+        self,
+        binding_id: str,
+        *,
+        delete_contributions: bool,
+        cleanup_failed: bool = False,
+    ) -> dict[str, Any] | None:
+        async with self._sf() as session:
+            binding = await session.get(SkillKnowledgeBindingRow, binding_id)
+            if binding is None:
+                return None
+            binding.active = False
+            binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
+            binding.updated_at = _now()
+            if delete_contributions:
+                await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
+                await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
+                await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
+                await self._retire_target_orphans(
+                    session,
+                    target_type=binding.target_type,
+                    target_id=binding.target_id,
+                )
+            await session.flush()
+            data = _dict(binding)
+            await session.commit()
+            return data
+
+    async def mark_skill_stale(self, skill_name: str) -> int:
+        async with self._sf() as session:
+            result = await session.execute(
+                SkillKnowledgeBindingRow.__table__.update()
+                .where(
+                    SkillKnowledgeBindingRow.skill_name == skill_name,
+                    SkillKnowledgeBindingRow.active.is_(True),
+                    SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
+                )
+                .values(status="stale", updated_at=_now())
+            )
+            await session.commit()
+            return int(result.rowcount or 0)
+
+
+def make_skill_knowledge_store(session_factory):
+    return SkillKnowledgeRepository(session_factory) if session_factory is not None else None

offline-backend-20260512/backend/packages/harness/deerflow/skill_knowledge/__init__.py

@@ -0,0 +1,10 @@
+"""Skill knowledge distillation primitives.
+
+The package is deliberately independent from ``app.*``. Gateway composition
+injects persistence and remote adapters at runtime.
+"""
+
+from deerflow.skill_knowledge.extractor import build_skill_ir
+from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
+
+__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]

offline-backend-20260512/backend/packages/harness/deerflow/skill_knowledge/extractor.py

@@ -0,0 +1,152 @@
+"""IR construction and validation.
+
+V1 intentionally starts with deterministic, evidence-backed extraction. The
+schema is compatible with a future model-backed extractor, while never letting
+unvalidated model output flow directly into persistence or graph writers.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from typing import Any
+
+from deerflow.skill_knowledge.scanner import ScanResult
+
+EXTRACTOR_VERSION = "skill-knowledge-v1"
+PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
+PREDICATE_PATTERNS = {
+    "depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^，。；;\n`\"”]{2,80})", re.I),
+    "uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^，。；;\n`\"”]{2,80})", re.I),
+    "produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^，。；;\n`\"”]{2,80})", re.I),
+    "consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^，。；;\n`\"”]{2,80})", re.I),
+}
+
+
+def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
+    line_start = text.count("\n", 0, start) + 1
+    line_end = text.count("\n", 0, end) + 1
+    quote = " ".join(text[start:end].split())[:240]
+    return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
+
+
+def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
+    entities: list[dict[str, Any]] = [
+        {
+            "local_id": "ent_skill",
+            "type": "skill",
+            "name": display_name or scan.skill_name,
+            "aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
+            "description": description,
+            "attributes": {},
+            "evidence": [],
+            "confidence": 1.0,
+        }
+    ]
+    relations: list[dict[str, Any]] = []
+    by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
+
+    for file in scan.files:
+        if not file.text:
+            continue
+        for predicate, pattern in PREDICATE_PATTERNS.items():
+            for match in pattern.finditer(file.text):
+                name = match.group(1).strip(" `\"'“”()[]")
+                if not name or len(name) > 80:
+                    continue
+                key = ("concept", name.casefold())
+                entity_id = by_key.get(key)
+                if entity_id is None:
+                    entity_id = f"ent_{len(entities):04d}"
+                    by_key[key] = entity_id
+                    entities.append(
+                        {
+                            "local_id": entity_id,
+                            "type": "concept",
+                            "name": name,
+                            "aliases": [],
+                            "description": "从技能业务说明中抽取的知识对象",
+                            "attributes": {},
+                            "evidence": [_evidence(file.path, file.text, match.start(), match.end())],
+                            "confidence": 0.82,
+                        }
+                    )
+                relations.append(
+                    {
+                        "local_id": f"rel_{len(relations):04d}",
+                        "source_local_id": "ent_skill",
+                        "predicate": predicate,
+                        "target_local_id": entity_id,
+                        "description": " ".join(match.group(0).split())[:240],
+                        "relation_source": "explicit_text",
+                        "evidence": [_evidence(file.path, file.text, match.start(), match.end())],
+                        "confidence": 0.82,
+                    }
+                )
+        for ref in file.references:
+            name = ref["target_path"]
+            key = ("file", name.casefold())
+            entity_id = by_key.get(key)
+            if entity_id is None:
+                entity_id = f"ent_{len(entities):04d}"
+                by_key[key] = entity_id
+                entities.append(
+                    {
+                        "local_id": entity_id,
+                        "type": "file",
+                        "name": name,
+                        "aliases": [],
+                        "description": "技能文档引用的资料文件",
+                        "attributes": {},
+                        "evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
+                        "confidence": 1.0,
+                    }
+                )
+            relations.append(
+                {
+                    "local_id": f"rel_{len(relations):04d}",
+                    "source_local_id": "ent_skill",
+                    "predicate": "references",
+                    "target_local_id": entity_id,
+                    "description": f"{file.path} 引用 {name}",
+                    "relation_source": "deterministic_structure",
+                    "evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
+                    "confidence": 1.0,
+                }
+            )
+
+    title = display_name or scan.skill_name
+    summaries = [f"- **{item.title}**（`{item.path}`）" for item in scan.files[:100]]
+    content = "\n".join(
+        [
+            f"# {title}",
+            "",
+            description or "由技能知识文件安全归纳生成。",
+            "",
+            "## 知识资料",
+            "",
+            *(summaries or ["- 暂无可归纳知识文件"]),
+            "",
+            f"> 源摘要：`{scan.source_digest}`；代码与脚本文件未读取。",
+        ]
+    )
+    return {
+        "extractor_version": EXTRACTOR_VERSION,
+        "prompt_hash": PROMPT_HASH,
+        "model_name": "deterministic-v1",
+        "skill": {
+            "name": scan.skill_name,
+            "display_name": title,
+            "description": description,
+            "source_root": scan.source_root,
+            "source_digest": scan.source_digest,
+        },
+        "files": scan.manifest,
+        "entities": entities,
+        "relations": relations,
+        "wiki": {
+            "skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
+            "pages": [],
+        },
+        "safety": {"blocked_files": scan.blocked_files},
+    }

offline-backend-20260512/backend/packages/harness/deerflow/skill_knowledge/graph_writer.py

@@ -0,0 +1,115 @@
+"""Guarded graph-write plans.
+
+No caller can submit Cypher. The writer accepts only normalized node/edge
+records and executes fixed parameterized templates when explicitly enabled.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+class GraphTransaction(Protocol):
+    async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
+
+
+@dataclass(frozen=True, slots=True)
+class GraphWritePlan:
+    scope: str
+    nodes: tuple[dict[str, Any], ...]
+    edges: tuple[dict[str, Any], ...]
+
+
+NODE_QUERY = """
+MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
+SET n.display_name = $display_name, n.entity_type = $entity_type,
+    n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
+    n.confidence = $confidence, n.updated_at = datetime()
+""".strip()
+EDGE_QUERY = """
+MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
+MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
+MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
+SET r.predicate = $predicate, r.binding_id = $binding_id,
+    r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
+    r.evidence = $evidence, r.updated_at = datetime()
+""".strip()
+
+
+def _key(*parts: str) -> str:
+    return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
+
+
+def plan_from_ir(
+    ir: dict[str, Any],
+    *,
+    target_type: str,
+    target_id: str,
+    binding_id: str,
+    snapshot_id: str,
+    job_id: str,
+) -> GraphWritePlan:
+    """Build a stable write plan; callers cannot supply executable Cypher."""
+    scope = f"{target_type}:{target_id}"
+    local_keys: dict[str, str] = {}
+    nodes: list[dict[str, Any]] = []
+    for entity in ir.get("entities") or []:
+        entity_type = str(entity.get("type") or "concept")
+        normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
+        if not normalized_name:
+            continue
+        entity_key = _key(entity_type, normalized_name)
+        local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
+        nodes.append(
+            {
+                "entity_key": entity_key,
+                "display_name": str(entity.get("name") or normalized_name),
+                "entity_type": entity_type,
+                "binding_id": binding_id,
+                "snapshot_id": snapshot_id,
+                "job_id": job_id,
+                "confidence": float(entity.get("confidence") or 1.0),
+            }
+        )
+    edges: list[dict[str, Any]] = []
+    for relation in ir.get("relations") or []:
+        source_key = local_keys.get(str(relation.get("source_local_id") or ""))
+        target_key = local_keys.get(str(relation.get("target_local_id") or ""))
+        evidence = list(relation.get("evidence") or [])
+        confidence = float(relation.get("confidence") or 0)
+        predicate = str(relation.get("predicate") or "related_to")
+        if not source_key or not target_key or confidence < 0.75 or not evidence:
+            continue
+        edges.append(
+            {
+                "edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
+                "source_key": source_key,
+                "target_key": target_key,
+                "predicate": predicate,
+                "binding_id": binding_id,
+                "snapshot_id": snapshot_id,
+                "job_id": job_id,
+                "confidence": confidence,
+                "evidence": evidence,
+            }
+        )
+    return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
+
+
+def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
+    """Return Neo4j transactional HTTP statements using only fixed templates."""
+    statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
+    statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
+    return statements
+
+
+async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
+    if not enabled:
+        return {"nodes": 0, "edges": 0}
+    for node in plan.nodes:
+        await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
+    for edge in plan.edges:
+        await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
+    return {"nodes": len(plan.nodes), "edges": len(plan.edges)}

offline-backend-20260512/backend/packages/harness/deerflow/skill_knowledge/merge.py

@@ -0,0 +1,31 @@
+"""Portable entity/relation normalization used by both knowledge products."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+
+def normalize_name(value: str) -> str:
+    text = unicodedata.normalize("NFKC", value or "").strip().casefold()
+    text = text.strip("`'\"“”‘’《》〈〉[](){}")
+    text = re.sub(r"\s+", " ", text)
+    return text
+
+
+def normalize_predicate(value: str) -> str:
+    normalized = normalize_name(value).replace(" ", "_")
+    aliases = {
+        "依赖": "depends_on",
+        "需要": "depends_on",
+        "requires": "depends_on",
+        "调用": "uses",
+        "使用": "uses",
+        "uses": "uses",
+        "生成": "produces",
+        "输出": "produces",
+        "读取": "consumes",
+        "输入": "consumes",
+        "引用": "references",
+    }
+    return aliases.get(normalized, normalized or "related_to")

offline-backend-20260512/backend/packages/harness/deerflow/skill_knowledge/scanner.py

@@ -0,0 +1,351 @@
+"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import json
+import re
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+CODE_SUFFIXES = {
+    ".py",
+    ".pyc",
+    ".pyo",
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".cjs",
+    ".sh",
+    ".bash",
+    ".zsh",
+    ".fish",
+    ".bat",
+    ".cmd",
+    ".ps1",
+    ".psm1",
+    ".java",
+    ".kt",
+    ".kts",
+    ".go",
+    ".rs",
+    ".c",
+    ".cc",
+    ".cpp",
+    ".h",
+    ".hpp",
+    ".cs",
+    ".php",
+    ".rb",
+    ".pl",
+    ".lua",
+    ".swift",
+    ".scala",
+    ".sql",
+    ".vue",
+    ".svelte",
+}
+TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
+BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
+EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
+EXCLUDED_FILENAMES = {
+    "package-lock.json",
+    "pnpm-lock.yaml",
+    "yarn.lock",
+    "bun.lockb",
+    "poetry.lock",
+    "uv.lock",
+    "pipfile.lock",
+    "cargo.lock",
+    "composer.lock",
+}
+SENSITIVE_NAME_RE = re.compile(
+    r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
+    re.IGNORECASE,
+)
+SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
+PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
+FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
+MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
+MAX_TEXT_BYTES = 2 * 1024 * 1024
+MAX_ARCHIVE_FILES = 1_000
+MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
+MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
+
+
+@dataclass(slots=True)
+class ScannedFile:
+    path: str
+    kind: str
+    sha256: str
+    size: int
+    text: str = ""
+    title: str = ""
+    references: list[dict[str, Any]] = field(default_factory=list)
+    safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
+
+    def manifest(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "kind": self.kind,
+            "sha256": self.sha256,
+            "size": self.size,
+            "text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
+            "title": self.title,
+            "references": self.references,
+            "safety": self.safety,
+        }
+
+
+@dataclass(slots=True)
+class ScanResult:
+    skill_name: str
+    source_root: str
+    source_digest: str
+    files: list[ScannedFile]
+    blocked_files: list[dict[str, str]]
+    skipped_code_count: int
+    skipped_other_count: int
+
+    @property
+    def manifest(self) -> list[dict[str, Any]]:
+        return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
+
+    @property
+    def counts(self) -> dict[str, int]:
+        return {
+            "knowledge_files": len(self.files),
+            "blocked_files": len(self.blocked_files),
+            "skipped_code_files": self.skipped_code_count,
+            "skipped_other_files": self.skipped_other_count,
+        }
+
+
+def _safe_relative(path: Path, root: Path) -> str:
+    resolved = path.resolve()
+    resolved_root = root.resolve()
+    try:
+        relative = resolved.relative_to(resolved_root)
+    except ValueError as exc:
+        raise ValueError("Skill file escapes its source root") from exc
+    return relative.as_posix()
+
+
+def _strip_markdown_code(text: str) -> str:
+    return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
+
+
+def _read_text(path: Path, suffix: str) -> str:
+    raw = path.read_bytes()
+    if len(raw) > MAX_TEXT_BYTES:
+        raise ValueError("text file exceeds 2 MiB parsing limit")
+    text = raw.decode("utf-8-sig", errors="replace")
+    if suffix in {".md", ".mdx"}:
+        return _strip_markdown_code(text)
+    if suffix in {".csv", ".tsv"}:
+        delimiter = "\t" if suffix == ".tsv" else ","
+        rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
+        return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
+    if suffix == ".json":
+        try:
+            return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
+        except json.JSONDecodeError:
+            return text
+    return text
+
+
+def _title(text: str, fallback: str) -> str:
+    match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
+    return match.group(1).strip() if match else fallback
+
+
+def _kind(suffix: str) -> str:
+    return {
+        ".md": "markdown",
+        ".mdx": "markdown",
+        ".txt": "text",
+        ".csv": "table",
+        ".tsv": "table",
+        ".xlsx": "spreadsheet",
+        ".xls": "spreadsheet",
+        ".pdf": "pdf",
+        ".docx": "office",
+        ".pptx": "office",
+        ".png": "image",
+        ".jpg": "image",
+        ".jpeg": "image",
+        ".webp": "image",
+        ".gif": "image",
+        ".bmp": "image",
+        ".zip": "archive",
+    }.get(suffix, "structured_data")
+
+
+def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
+    files: list[ScannedFile] = []
+    blocked: list[dict[str, str]] = []
+    skipped_code = 0
+    skipped_other = 0
+    with zipfile.ZipFile(path) as archive:
+        members = archive.infolist()
+        if len(members) > MAX_ARCHIVE_FILES:
+            return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
+        total_size = sum(member.file_size for member in members)
+        if total_size > MAX_ARCHIVE_TOTAL_BYTES:
+            return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
+        for member in members:
+            member_path = Path(member.filename.replace("\\", "/"))
+            virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
+            unix_mode = member.external_attr >> 16
+            if member.is_dir():
+                continue
+            if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
+                blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
+                continue
+            suffix = member_path.suffix.lower()
+            lower_name = member_path.name.lower()
+            if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
+                skipped_code += 1
+                continue
+            if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
+                blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
+                continue
+            if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
+                skipped_other += 1
+                continue
+            raw = archive.read(member)
+            text = raw.decode("utf-8-sig", errors="replace")
+            if suffix in {".md", ".mdx"}:
+                text = _strip_markdown_code(text)
+            if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
+                blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
+                continue
+            files.append(
+                ScannedFile(
+                    path=virtual_path,
+                    kind=_kind(suffix),
+                    sha256=hashlib.sha256(raw).hexdigest(),
+                    size=len(raw),
+                    text=text,
+                    title=_title(text, member_path.stem),
+                )
+            )
+    return files, blocked, skipped_code, skipped_other
+
+
+def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
+    """Scan one already-resolved skill directory without following symlinks."""
+
+    root = skill_root.resolve()
+    if not root.is_dir():
+        raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
+    files: list[ScannedFile] = []
+    blocked: list[dict[str, str]] = []
+    skipped_code = 0
+    skipped_other = 0
+
+    for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
+        if path.is_symlink() or not path.is_file():
+            continue
+        relative = _safe_relative(path, root)
+        if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
+            continue
+        lower_name = path.name.lower()
+        suffix = path.suffix.lower()
+        if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
+            skipped_code += 1
+            continue
+        if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
+            blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
+            continue
+        if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
+            skipped_other += 1
+            continue
+
+        # Never upload an archive wholesale: it may contain excluded code or a
+        # blocked secret. Only independently validated knowledge members enter
+        # the manifest and digest.
+        if suffix == ".zip":
+            try:
+                nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
+                files.extend(nested)
+                blocked.extend(nested_blocked)
+                skipped_code += nested_code
+                skipped_other += nested_other
+            except (OSError, zipfile.BadZipFile):
+                blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
+            continue
+
+        raw = path.read_bytes()
+        digest = hashlib.sha256(raw).hexdigest()
+        text = ""
+        warnings: list[str] = []
+        if suffix in TEXT_SUFFIXES:
+            try:
+                text = _read_text(path, suffix)
+            except ValueError as exc:
+                warnings.append(str(exc))
+            if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
+                blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
+                continue
+        references = []
+        if suffix in {".md", ".mdx"}:
+            for target in MARKDOWN_LINK_RE.findall(text):
+                normalized = target.replace("\\", "/").lstrip("./")
+                if normalized:
+                    references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
+        files.append(
+            ScannedFile(
+                path=relative,
+                kind=_kind(suffix),
+                sha256=digest,
+                size=len(raw),
+                text=text,
+                title=_title(text, path.stem),
+                references=references,
+                safety={"status": "allowed", "warnings": warnings},
+            )
+        )
+
+    digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
+    source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
+    return ScanResult(
+        skill_name=skill_name,
+        source_root=str(root),
+        source_digest=source_digest,
+        files=files,
+        blocked_files=blocked,
+        skipped_code_count=skipped_code,
+        skipped_other_count=skipped_other,
+    )
+
+
+async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
+    """Extract Office/PDF/spreadsheet text without modifying the skill tree."""
+
+    from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
+
+    allowed: list[ScannedFile] = []
+    root = Path(scan.source_root)
+    for item in scan.files:
+        if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
+            text = await extract_file_to_markdown(root / item.path)
+            if text:
+                text = _strip_markdown_code(text)
+                if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
+                    scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
+                    continue
+                item.text = text
+                item.title = _title(text, item.title)
+            else:
+                item.safety["warnings"].append("文档转换不可用，仅保留原始附件")
+        allowed.append(item)
+    scan.files = allowed
+    digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
+    scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
+    return scan

offline-backend-20260512/backend/packages/harness/deerflow/utils/file_conversion.py

@@ -167,6 +167,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
         return None
 
 
+async def extract_file_to_markdown(file_path: Path) -> str | None:
+    """Extract a supported document to Markdown without creating a sidecar file.
+
+    Knowledge-ingest scanners use this variant because running a read-only
+    distillation task must never add generated ``.md`` files to a live skill.
+    Conversion always runs off the event loop; Office/PDF parsers can be CPU
+    intensive even for deceptively small inputs.
+    """
+
+    if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
+        return None
+    try:
+        return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
+    except Exception as exc:
+        logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
+        return None
+
+
 # Regex for bold-only lines that look like section headings.
 # Targets SEC filing structural headings that pymupdf4llm renders as **bold**
 # rather than # Markdown headings (because they use same font size as body text,