feat: improve assistant knowledge import and database compatibility

This commit is contained in:
吉鑫升 committed 2026-09-02 17:55:17 +08:00
1 parent fbd8dec48f
commit 6b724bd00d
51 files changed
+4393 -1021

No files matched your search

@@ -0,0 +1,109 @@
"""File-backed assistant knowledge import/export packages.
The assistant knowledge base is fed by WeKnora-exported data. Keep that
boundary explicit: a WeKnora export is first materialized as a durable JSONL
package, then imported asynchronously from that package. JSONL keeps large
exports append-friendly and avoids treating the HTTP request body as the job
state.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
PACKAGE_VERSION = 1
PACKAGE_RECORD_TYPES = {
"source",
"wiki_page",
"document",
"chunk",
"vector",
"entity",
"relation",
"graph",
}
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
return root / f"{safe}.jsonl"
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
handle.write("\n")
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
path.parent.mkdir(parents=True, exist_ok=True)
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
tmp_path = path.with_suffix(path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
write_jsonl_record(
handle,
"source",
{
"package_version": PACKAGE_VERSION,
},
)
counts["source"] += 1
for record_type, payload in records:
if record_type not in PACKAGE_RECORD_TYPES:
continue
write_jsonl_record(handle, record_type, payload)
counts[record_type] = counts.get(record_type, 0) + 1
tmp_path.replace(path)
return counts
def read_package(path: Path) -> dict[str, Any]:
package: dict[str, Any] = {
"source": {},
"wiki_pages": [],
"documents": [],
"chunks": [],
"vectors": [],
"entities": [],
"relations": [],
"graph": {"nodes": [], "edges": []},
}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
record = json.loads(line)
if not isinstance(record, dict):
continue
record_type = str(record.get("record_type") or "")
payload = record.get("payload")
if not isinstance(payload, dict):
continue
if record_type == "source":
package["source"] = {**package.get("source", {}), **payload}
elif record_type == "wiki_page":
package["wiki_pages"].append(payload)
elif record_type == "document":
package["documents"].append(payload)
elif record_type == "chunk":
package["chunks"].append(payload)
elif record_type == "vector":
package["vectors"].append(payload)
elif record_type == "entity":
package["entities"].append(payload)
elif record_type == "relation":
package["relations"].append(payload)
elif record_type == "graph":
nodes = payload.get("nodes")
edges = payload.get("edges")
if isinstance(nodes, list):
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
if isinstance(edges, list):
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
for key, value in payload.items():
if key not in {"nodes", "edges"}:
package["graph"][key] = value
return package
@@ -11,10 +11,15 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Gauss mode: application ORM data uses the official openGauss SQLAlchemy
dialect over asyncpg. LangGraph's PostgreSQL checkpointer is intentionally not
reused for GaussDB; configure the legacy ``checkpointer:`` section separately
(normally sqlite).
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
@@ -42,24 +47,40 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "gauss", "mysql"] = Field(
default="memory",
description=(
"Storage backend for application data. 'memory' for development "
"(no persistence across restarts), 'sqlite' for single-node "
"deployment, 'postgres' for production multi-node deployment, "
"'gauss' for openGauss/GaussDB business tables, and 'mysql' for "
"MySQL-backed business tables."
),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
),
)
gauss_url: str = Field(
default="",
description=(
"openGauss/GaussDB connection URL for the application ORM. "
"Use $GAUSS_DATABASE_URL in config.yaml to reference .env. "
"Example: opengauss+asyncpg://user:pass@host:26000/deerflow. "
"Plain opengauss:// or PostgreSQL-style URLs are normalized to the "
"official asynchronous openGauss SQLAlchemy dialect."
),
)
mysql_url: str = Field(
default="",
description=(
@@ -75,26 +96,26 @@ class DatabaseConfig(BaseModel):
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/gauss/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"Extra connections allowed beyond pool_size under load (postgres/gauss/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"Seconds a caller waits for a free pooled connection before erroring (postgres/gauss/mysql only). "
"SQLAlchemy default is 30."
),
)
pool_recycle: int = Field(
default=1800,
description=(
"Recycle pooled connections after this many seconds (postgres/mysql only). "
"Recycle pooled connections after this many seconds (postgres/gauss/mysql only). "
"Prevents stale-connection errors when a firewall, pgbouncer, or MySQL "
"wait_timeout silently drops idle connections. pool_pre_ping catches most "
"cases, but pool_recycle is the belt-and-suspenders safety net. "
@@ -133,12 +154,38 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "gauss":
url = self.gauss_url.strip()
if not url:
raise ValueError("database.gauss_url is required for backend='gauss'")
# GaussDB speaks the PostgreSQL wire protocol, but its server
# version string and DDL capabilities are not identical to
# PostgreSQL. Always select the official openGauss dialect instead
# of silently sending the URL through SQLAlchemy's generic PG
# dialect (the latter commonly connects and then fails while
# SQLAlchemy initializes/creates the schema).
for prefix in (
"opengauss://",
"gaussdb://",
"postgres://",
"postgresql://",
"postgresql+asyncpg://",
):
if url.startswith(prefix):
return "opengauss+asyncpg://" + url[len(prefix) :]
if url.startswith("opengauss+asyncpg://"):
return url
raise ValueError(
"database.gauss_url must use opengauss://, "
"opengauss+asyncpg://, or a PostgreSQL-style URL"
)
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
@@ -73,7 +73,7 @@ class WeKnoraDirectGraphConfig(BaseModel):
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-m3"
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
dimensions: int | None = Field(
@@ -494,7 +494,7 @@ class WeKnoraClient:
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 500))},
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
)
)
if not isinstance(value, dict):
@@ -1,7 +1,37 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from __future__ import annotations
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from deerflow.integrations.weknora.local_index.retrieval import (
WikiRetrievalService,
is_fully_vectorized,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = [
"WikiIndexError",
"WikiRetrievalService",
"WikiVectorSearchService",
"WikiSyncService",
"is_fully_vectorized",
]
def __getattr__(name: str) -> Any:
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
from deerflow.integrations.weknora.local_index import retrieval
return getattr(retrieval, name)
if name in {"WikiIndexError", "WikiVectorSearchService"}:
from deerflow.integrations.weknora.local_index import search
return getattr(search, name)
if name == "WikiSyncService":
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
return WikiSyncService
raise AttributeError(name)
@@ -1,10 +1,10 @@
"""ORM model for the single global DeerFlow assistant knowledge base."""
"""ORM model for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, UniqueConstraint
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -19,7 +19,12 @@ class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# ``global`` is the full-library aggregation base. ``custom`` bases are
# user-created assistant knowledge bases fed from WeKnora export packages.
# Nullable keeps startup column reconciliation safe for existing databases.
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(64), default="active")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
@@ -91,6 +96,32 @@ class AssistantKnowledgeWikiRevisionRow(Base):
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeVectorRow(Base):
__tablename__ = "assistant_knowledge_vectors"
__table_args__ = (
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
Index("ix_assistant_knowledge_vector_page", "page_id"),
Index("ix_assistant_knowledge_vector_source", "source_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
@@ -1,15 +1,21 @@
"""Repository for the single global assistant knowledge base."""
"""Repository for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
import base64
import binascii
import difflib
import hashlib
import logging
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
import numpy as np
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector, normalize_vector
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
@@ -18,11 +24,24 @@ from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeVectorRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
GLOBAL_BASE_NAME = "知识梳理总库"
logger = logging.getLogger(__name__)
VECTOR_FIELD_NAMES = (
"embedding",
"vector",
"embedding_vector",
"content_vector",
"dense_vector",
"dense",
)
def _now() -> datetime:
return datetime.now(UTC)
@@ -35,31 +54,246 @@ def _dict(row: Any) -> dict[str, Any]:
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
if isinstance(row, AssistantKnowledgeBaseRow):
raw_status = str(data.get("status") or "")
data["status"] = "active" if raw_status.startswith("active") else raw_status
data["base_kind"] = data.get("base_kind") or "custom"
data["is_global"] = data["base_kind"] == "global"
data["description"] = data.get("description") or ""
return data
def _stable_slug_part(value: Any) -> str:
raw = str(value or "").strip()
if not raw:
raw = str(uuid4())
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
def _content_hash(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _first_vector_value(raw: dict[str, Any]) -> Any:
for key in VECTOR_FIELD_NAMES:
value = raw.get(key)
if value is not None:
return value
vectors = raw.get("vectors")
if isinstance(vectors, dict):
for key in VECTOR_FIELD_NAMES:
value = vectors.get(key)
if value is not None:
return value
return None
def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
blob64 = raw.get("vector_blob_base64") or raw.get("embedding_blob_base64")
dimensions = raw.get("embedding_dimensions") or raw.get("dimensions")
if blob64 and dimensions:
try:
decoded = base64.b64decode(str(blob64), validate=True)
vector = decode_vector(decoded, int(dimensions))
return encode_vector(vector)
except (ValueError, TypeError, binascii.Error):
return None
value = _first_vector_value(raw)
if value is None:
return None
if isinstance(value, dict):
value = _first_vector_value(value)
if not isinstance(value, (list, tuple)):
return None
try:
return encode_vector(value, dimensions=int(dimensions) if dimensions else None)
except (ValueError, TypeError):
return None
class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_base(self) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def _base_with_stats(
self,
session: AsyncSession,
row: AssistantKnowledgeBaseRow,
) -> dict[str, Any]:
data = _dict(row)
data["source_count"] = int(
(
await session.execute(
select(func.count(AssistantKnowledgeSourceRow.id)).where(
AssistantKnowledgeSourceRow.base_id == row.id,
AssistantKnowledgeSourceRow.status != "deleted",
)
)
).scalar_one()
or 0
)
data["page_count"] = int(
(
await session.execute(
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
AssistantKnowledgeWikiPageRow.base_id == row.id,
AssistantKnowledgeWikiPageRow.status == "active",
)
)
).scalar_one()
or 0
)
data["entity_count"] = int(
(
await session.execute(
select(func.count(AssistantKnowledgeEntityRow.id)).where(
AssistantKnowledgeEntityRow.base_id == row.id,
AssistantKnowledgeEntityRow.status == "active",
)
)
).scalar_one()
or 0
)
data["relation_count"] = int(
(
await session.execute(
select(func.count(AssistantKnowledgeRelationRow.id)).where(
AssistantKnowledgeRelationRow.base_id == row.id,
AssistantKnowledgeRelationRow.status == "active",
)
)
).scalar_one()
or 0
)
return data
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
async def get_base(self, base_id: str | None = None) -> dict[str, Any] | None:
async with self._sf() as session:
if base_id:
row = (
await session.execute(
select(AssistantKnowledgeBaseRow).where(
AssistantKnowledgeBaseRow.id == base_id,
AssistantKnowledgeBaseRow.status != "deleted",
)
)
).scalar_one_or_none()
return await self._base_with_stats(session, row) if row else None
row = (
await session.execute(
select(AssistantKnowledgeBaseRow)
.where(
AssistantKnowledgeBaseRow.status != "deleted",
AssistantKnowledgeBaseRow.base_kind == "global",
)
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
.limit(1)
)
).scalar_one_or_none()
if row is None:
# Backward compatibility: earlier builds had a single active row
# and no ``base_kind``. Promote the first surviving base into the
# global full-library aggregation base.
row = (
await session.execute(
select(AssistantKnowledgeBaseRow)
.where(AssistantKnowledgeBaseRow.status != "deleted")
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
.limit(1)
)
).scalar_one_or_none()
if row is not None:
row.base_kind = "global"
if not row.description:
row.description = "默认全库沉淀库,汇聚所有 WeKnora 导出的助手知识数据。"
row.updated_at = _now()
await session.flush()
await session.commit()
return await self._base_with_stats(session, row) if row else None
async def list_bases(self) -> list[dict[str, Any]]:
await self.get_base()
async with self._sf() as session:
rows = (
await session.execute(
select(AssistantKnowledgeBaseRow)
.where(AssistantKnowledgeBaseRow.status != "deleted")
.order_by(
AssistantKnowledgeBaseRow.base_kind.desc(),
AssistantKnowledgeBaseRow.updated_at.desc(),
)
)
).scalars()
values = [await self._base_with_stats(session, row) for row in rows]
global_values = [item for item in values if item.get("is_global")]
custom_values = sorted(
[item for item in values if not item.get("is_global")],
key=lambda item: item.get("updated_at") or "",
reverse=True,
)
return global_values + custom_values
async def initialize(self, *, created_by: str | None, name: str = GLOBAL_BASE_NAME) -> dict[str, Any]:
existing = await self.get_base()
if existing:
return existing
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
row = AssistantKnowledgeBaseRow(
id=str(uuid4()),
name=name.strip() or GLOBAL_BASE_NAME,
description="默认全库沉淀库,汇聚所有 WeKnora 导出的助手知识数据。",
base_kind="global",
status="active",
created_by=created_by,
created_at=_now(),
updated_at=_now(),
)
async with self._sf() as session:
session.add(row)
await session.flush()
data = _dict(row)
data = await self._base_with_stats(session, row)
await session.commit()
return data
async def create_base(
self,
*,
created_by: str | None,
name: str,
description: str = "",
) -> dict[str, Any]:
row_id = str(uuid4())
row = AssistantKnowledgeBaseRow(
id=row_id,
name=name.strip(),
description=description.strip(),
base_kind="custom",
# Some upgraded SQLite/MySQL databases may still carry the old
# unique(status) constraint from the single-base implementation and
# its original VARCHAR(24) length. Use a compact per-row active
# marker internally and normalize it in API responses.
status=f"active:{row_id[:17]}",
created_by=created_by,
created_at=_now(),
updated_at=_now(),
)
async with self._sf() as session:
session.add(row)
await session.flush()
data = await self._base_with_stats(session, row)
await session.commit()
return data
async def _ensure_base(self, base_id: str | None, *, created_by: str | None) -> dict[str, Any]:
if base_id == "global":
return await self.initialize(created_by=created_by)
if base_id:
base = await self.get_base(base_id)
if base is None:
raise ValueError("助手知识库不存在")
return base
return await self.initialize(created_by=created_by)
async def _upsert_source(
self,
session: AsyncSession,
@@ -181,8 +415,56 @@ class AssistantKnowledgeRepository:
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
base_id: str | None = None,
mirror_to_global: bool = True,
) -> dict[str, Any]:
base = await self.initialize(created_by=created_by)
base = await self._ensure_base(base_id, created_by=created_by)
result = await self._import_package_into_base(
base=base,
package=package,
source_type=source_type,
source_key=source_key,
source_name=source_name,
trigger=trigger,
created_by=created_by,
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
if mirror_to_global and not base.get("is_global"):
global_base = await self.initialize(created_by=created_by)
mirror = await self._import_package_into_base(
base=global_base,
package=package,
source_type=source_type,
source_key=source_key,
source_name=source_name,
trigger=f"{trigger}:global_mirror"[:48],
created_by=created_by,
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
result["mirrored_global_base_id"] = global_base["id"]
result["mirrored_global_job_id"] = mirror["id"]
return result
async def create_queued_import_job(
self,
*,
source_type: str,
source_key: str,
source_name: str,
trigger: str,
created_by: str | None,
metadata: dict[str, Any] | None = None,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
base_id: str | None = None,
phase: str = "queued",
) -> dict[str, Any]:
base = await self._ensure_base(base_id, created_by=created_by)
now = _now()
async with self._sf() as session:
source = await self._upsert_source(
@@ -191,22 +473,18 @@ class AssistantKnowledgeRepository:
source_type=source_type,
source_key=source_key,
source_name=source_name,
metadata=package.get("source") or {},
metadata=metadata or {},
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
# A re-import replaces this source's prior contributions while retaining
# canonical entities/relations that are still supported by other sources.
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
job = AssistantKnowledgeImportJobRow(
id=str(uuid4()),
base_id=base["id"],
source_id=source.id,
trigger=trigger,
status="running",
phase="aligning_entities",
status="queued",
phase=phase,
counts_json={},
created_by=created_by,
created_at=now,
@@ -214,6 +492,166 @@ class AssistantKnowledgeRepository:
session.add(job)
await session.flush()
source.last_import_job_id = job.id
result = _dict(job)
await session.commit()
return result
async def update_import_job(
self,
job_id: str,
*,
status: str | None = None,
phase: str | None = None,
counts: dict[str, Any] | None = None,
error: str | None = None,
completed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(AssistantKnowledgeImportJobRow, job_id)
if job is None:
return None
if status is not None:
job.status = status
if phase is not None:
job.phase = phase
if counts is not None:
job.counts_json = counts
if error is not None:
job.last_error_message = error[:2048]
elif status not in {None, "failed"}:
job.last_error_message = None
if completed:
job.completed_at = _now()
await session.flush()
result = _dict(job)
await session.commit()
return result
async def import_package_for_job(
self,
job_id: str,
*,
package: dict[str, Any],
created_by: str | None,
mirror_to_global: bool = True,
) -> dict[str, Any]:
async with self._sf() as session:
job = await session.get(AssistantKnowledgeImportJobRow, job_id)
if job is None:
raise ValueError("导入任务不存在")
source = await session.get(AssistantKnowledgeSourceRow, job.source_id)
base_row = await session.get(AssistantKnowledgeBaseRow, job.base_id)
if source is None or base_row is None:
raise ValueError("导入任务来源不存在")
base = _dict(base_row)
source_type = source.source_type
source_key = source.source_key
source_name = source.source_name
source_id = source.id
trigger = job.trigger
skill_name = source.skill_name
mapping_id = source.knowledge_base_mapping_id
upload_batch_id = source.upload_batch_id
result = await self._import_package_into_base(
base=base,
package=package,
source_type=source_type,
source_key=source_key,
source_name=source_name,
trigger=trigger,
created_by=created_by,
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
existing_source_id=source_id,
existing_job_id=job_id,
)
if mirror_to_global and not base.get("is_global"):
global_base = await self.initialize(created_by=created_by)
mirror = await self._import_package_into_base(
base=global_base,
package=package,
source_type=source_type,
source_key=source_key,
source_name=source_name,
trigger=f"{trigger}:global_mirror"[:48],
created_by=created_by,
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
result["mirrored_global_base_id"] = global_base["id"]
result["mirrored_global_job_id"] = mirror["id"]
return result
async def _import_package_into_base(
self,
*,
base: dict[str, Any],
package: dict[str, Any],
source_type: str,
source_key: str,
source_name: str,
trigger: str,
created_by: str | None,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
existing_source_id: str | None = None,
existing_job_id: str | None = None,
) -> dict[str, Any]:
now = _now()
async with self._sf() as session:
if existing_source_id:
source = await session.get(AssistantKnowledgeSourceRow, existing_source_id)
if source is None or source.base_id != base["id"]:
raise ValueError("导入任务来源不存在")
source.source_name = source_name
source.status = "active"
source.metadata_json = package.get("source") or {}
source.updated_at = now
else:
source = await self._upsert_source(
session,
base_id=base["id"],
source_type=source_type,
source_key=source_key,
source_name=source_name,
metadata=package.get("source") or {},
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
# A re-import replaces this source's prior contributions while retaining
# canonical entities/relations that are still supported by other sources.
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeVectorRow).where(AssistantKnowledgeVectorRow.source_id == source.id))
if existing_job_id:
job = await session.get(AssistantKnowledgeImportJobRow, existing_job_id)
if job is None or job.base_id != base["id"]:
raise ValueError("导入任务不存在")
job.source_id = source.id
job.status = "running"
job.phase = "aligning_entities"
job.last_error_message = None
job.completed_at = None
else:
job = AssistantKnowledgeImportJobRow(
id=str(uuid4()),
base_id=base["id"],
source_id=source.id,
trigger=trigger,
status="running",
phase="aligning_entities",
counts_json={},
created_by=created_by,
created_at=now,
)
session.add(job)
await session.flush()
source.last_import_job_id = job.id
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
@@ -363,6 +801,7 @@ class AssistantKnowledgeRepository:
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
pages = [page] if page else []
page_ids_by_slug: dict[str, str] = {}
for raw in pages:
if not isinstance(raw, dict):
continue
@@ -404,16 +843,139 @@ class AssistantKnowledgeRepository:
)
session.add(revision)
page.title = title
page.page_type = str(raw.get("page_type") or "source")
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
page_ids_by_slug[slug] = page.id
chunks = list(package.get("chunks") or [])
chunk_pages = 0
imported_vectors = 0
for index, raw in enumerate(chunks):
if not isinstance(raw, dict):
continue
content = str(
raw.get("content")
or raw.get("section_content")
or raw.get("text")
or raw.get("chunk_content")
or ""
).strip()
if not content:
continue
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
title = str(
raw.get("title")
or raw.get("knowledge_title")
or raw.get("filename")
or raw.get("knowledge_filename")
or f"{source_name} 片段 {index + 1}"
)
heading = str(raw.get("heading") or raw.get("section_title") or "")
summary = content[:240]
page_content = "\n".join(
item
for item in [
f"# {title}",
"",
f"- 来源:{source_name}",
f"- 文档ID:{document_id}" if document_id else "",
f"- 片段ID:{raw_chunk_id}" if raw_chunk_id else "",
f"- 标题层级:{heading}" if heading else "",
"",
content,
]
if item != ""
)
chunk_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"chunks/{source.id}/{_stable_slug_part(raw_chunk_id)}",
title=title,
page_type="chunk",
content=page_content,
summary=summary,
search_terms=[raw_chunk_id, document_id, heading],
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="chunk_import",
)
page_ids_by_slug[chunk_page.slug] = chunk_page.id
vector_blob = _vector_blob_from_payload(raw)
if vector_blob is not None:
blob, dimensions = vector_blob
session.add(
AssistantKnowledgeVectorRow(
id=str(uuid4()),
base_id=base["id"],
page_id=chunk_page.id,
source_id=source.id,
import_job_id=job.id,
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
source_chunk_id=raw_chunk_id,
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
heading=heading or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
embedding_model=str(raw.get("embedding_model") or raw.get("model") or ""),
embedding_fingerprint=str(raw.get("embedding_fingerprint") or ""),
embedding_dimensions=dimensions,
vector_blob=blob,
created_at=now,
)
)
imported_vectors += 1
chunk_pages += 1
for index, raw in enumerate(list(package.get("vectors") or [])):
if not isinstance(raw, dict):
continue
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
page_id = page_ids_by_slug.get(slug)
if not page_id:
continue
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
if not content:
continue
vector_blob = _vector_blob_from_payload(raw)
if vector_blob is None:
continue
blob, dimensions = vector_blob
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
session.add(
AssistantKnowledgeVectorRow(
id=str(uuid4()),
base_id=base["id"],
page_id=page_id,
source_id=source.id,
import_job_id=job.id,
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
source_chunk_id=raw_chunk_id or None,
section_index=int(raw.get("section_index") or index),
heading=str(raw.get("heading") or "") or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
embedding_model=str(raw.get("embedding_model") or raw.get("model") or ""),
embedding_fingerprint=str(raw.get("embedding_fingerprint") or ""),
embedding_dimensions=dimensions,
vector_blob=blob,
created_at=now,
)
)
imported_vectors += 1
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"documents": len(package.get("documents") or []),
"chunks": len(package.get("chunks") or []),
"chunks": len(chunks),
"chunk_pages": chunk_pages,
"vectors": imported_vectors,
"entities": len(raw_entities),
"relations": len(raw_relations),
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
@@ -439,8 +1001,24 @@ class AssistantKnowledgeRepository:
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
async def list_pages(
self,
*,
base_id: str | None = None,
query: str | None = None,
page_type: str | None = None,
limit: int = 100,
include_chunks: bool = False,
) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
if base is None:
return []
stmt = select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.status == "active",
)
if not include_chunks and not page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
if query:
needle = f"%{query.strip()}%"
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
@@ -450,15 +1028,178 @@ class AssistantKnowledgeRepository:
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_page(self, slug: str) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
async def search_pages(
self,
*,
base_ids: list[str],
query: str,
limit: int = 8,
include_chunks: bool = True,
embedding_client: Any | None = None,
) -> list[dict[str, Any]]:
resolved_base_ids: list[str] = []
for base_id in list(dict.fromkeys(base_ids)):
base = await self.get_base(base_id)
if base is not None:
resolved_base_ids.append(str(base["id"]))
if not resolved_base_ids:
return []
if embedding_client is not None and query.strip():
try:
vector_results = await self._search_pages_by_vector(
base_ids=resolved_base_ids,
query=query,
limit=limit,
include_chunks=include_chunks,
embedding_client=embedding_client,
)
if vector_results:
return vector_results
except Exception: # noqa: BLE001
logger.warning("Assistant knowledge vector search failed; falling back to keyword search", exc_info=True)
tokens = [token for token in query.strip().lower().split() if token]
needle = f"%{query.strip()}%" if query.strip() else "%"
stmt = select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id.in_(resolved_base_ids),
AssistantKnowledgeWikiPageRow.status == "active",
)
if not include_chunks:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
if query.strip():
stmt = stmt.where(
or_(
AssistantKnowledgeWikiPageRow.title.ilike(needle),
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
*[
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
for token in tokens[:8]
],
)
)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
async with self._sf() as session:
rows = [_dict(row) for row in (await session.execute(stmt)).scalars()]
def score(row: dict[str, Any]) -> float:
text = f"{row.get('title') or ''}\n{row.get('summary') or ''}\n{row.get('search_text') or row.get('content_markdown') or ''}".lower()
if not tokens:
return 0.0
value = 0.0
title = str(row.get("title") or "").lower()
for token in tokens:
if token in title:
value += 3.0
value += min(text.count(token), 8) * 0.25
if row.get("page_type") == "chunk":
value += 0.5
return value
rows.sort(key=score, reverse=True)
results = rows[:limit]
for row in results:
row["score"] = score(row)
return results
async def _search_pages_by_vector(
self,
*,
base_ids: list[str],
query: str,
limit: int,
include_chunks: bool,
embedding_client: Any,
) -> list[dict[str, Any]]:
raw_query_vector = await embedding_client.embed_query(query)
configured_dimensions = getattr(embedding_client, "dimensions", None)
query_vector = normalize_vector(raw_query_vector, dimensions=configured_dimensions)
dimensions = int(query_vector.size)
stmt = (
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow)
.join(
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
)
.where(
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
AssistantKnowledgeWikiPageRow.status == "active",
)
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
)
if not include_chunks:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
async with self._sf() as session:
rows = list((await session.execute(stmt)).all())
if not rows:
return []
tokens = [token for token in query.strip().lower().split() if token]
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
for vector, page in rows:
try:
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
except ValueError:
continue
score = float(np.dot(query_vector, stored))
if tokens:
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
ranked.append((score, vector, page))
ranked.sort(key=lambda item: item[0], reverse=True)
results: list[dict[str, Any]] = []
seen_pages: set[str] = set()
for score, vector, page in ranked:
if len(results) >= limit:
break
page_data = _dict(page)
page_data["score"] = score
page_data["content_markdown"] = vector.section_content or page_data.get("content_markdown") or ""
page_data["summary"] = page_data.get("summary") or (vector.section_content[:240] if vector.section_content else "")
page_data["retrieval_mode"] = "vector"
page_data["matched_section"] = {
"id": vector.id,
"section_index": int(vector.section_index),
"heading": vector.heading,
"content": vector.section_content,
"score": score,
"source_chunk_id": vector.source_chunk_id,
"embedding_model": vector.embedding_model,
"embedding_fingerprint": vector.embedding_fingerprint,
}
dedupe_key = f"{page.id}:{vector.section_index}:{vector.content_hash}"
if page.id in seen_pages and page.page_type != "chunk":
continue
seen_pages.add(page.id)
seen_pages.add(dedupe_key)
results.append(page_data)
return results
async def get_page(self, slug: str, *, base_id: str | None = None) -> dict[str, Any] | None:
base = await self.get_base(base_id)
if base is None:
return None
stmt = select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
AssistantKnowledgeWikiPageRow.status == "active",
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
async def list_revisions(self, slug: str, *, base_id: str | None = None) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
if base is None:
return []
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
page = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
)
)
).scalar_one_or_none()
if page is None:
return []
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
@@ -484,10 +1225,27 @@ class AssistantKnowledgeRepository:
)
return {"left": left, "right": right, "diff": diff}
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
async def rollback(
self,
*,
slug: str,
revision_id: str,
actor: str | None,
base_id: str | None = None,
) -> dict[str, Any] | None:
base = await self.get_base(base_id)
if base is None:
return None
now = _now()
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
page = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
)
)
).scalar_one_or_none()
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
if page is None or source is None or source.page_id != page.id:
return None
@@ -517,9 +1275,19 @@ class AssistantKnowledgeRepository:
await session.commit()
return data
async def list_sources(self) -> list[dict[str, Any]]:
async def list_sources(self, *, base_id: str | None = None) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
if base is None:
return []
async with self._sf() as session:
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
stmt = (
select(AssistantKnowledgeSourceRow)
.where(
AssistantKnowledgeSourceRow.base_id == base["id"],
AssistantKnowledgeSourceRow.status != "deleted",
)
.order_by(AssistantKnowledgeSourceRow.updated_at.desc())
)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
@@ -547,9 +1315,17 @@ class AssistantKnowledgeRepository:
result["contributions"] = [_dict(value) for value in contributions]
return result
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
async def list_jobs(self, limit: int = 100, *, base_id: str | None = None) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
if base is None:
return []
async with self._sf() as session:
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
stmt = (
select(AssistantKnowledgeImportJobRow)
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
.limit(limit)
)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None:
@@ -565,6 +1341,7 @@ class AssistantKnowledgeRepository:
source.status = "deleted"
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
await session.execute(delete(AssistantKnowledgeVectorRow).where(AssistantKnowledgeVectorRow.source_id == source_id))
pages = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
@@ -162,7 +162,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -216,10 +216,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
added column are created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -341,13 +341,13 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
@@ -365,11 +365,21 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend == "postgres":
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -409,7 +419,7 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
@@ -454,7 +464,7 @@ async def init_engine(
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
@@ -497,7 +507,7 @@ async def init_engine(
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s", backend)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -3,6 +3,7 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
"""Iterate vector rows for streaming export.
Implementations with a database backend should override this method so
very large vector packages do not need to be materialized in memory.
"""
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
for row in snapshot.get("rows") or []:
if isinstance(row, dict):
yield row
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@@ -2,6 +2,7 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -181,6 +182,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
batch = max(1, min(int(batch_size), 5000))
offset = 0
while True:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
.limit(batch)
.offset(offset)
)
async with self._sf() as session:
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
if not rows:
break
for row in rows:
yield row
if len(rows) < batch:
break
offset += len(rows)
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -0,0 +1,25 @@
"""Add assistant knowledge vector table.
Revision ID: 20260902_01
Revises: 20260901_01
Create Date: 2026-09-02
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import AssistantKnowledgeVectorRow
revision = "20260902_01"
down_revision = "20260901_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
AssistantKnowledgeVectorRow.__table__.create(op.get_bind(), checkfirst=True)
def downgrade() -> None:
AssistantKnowledgeVectorRow.__table__.drop(op.get_bind(), checkfirst=True)
@@ -43,6 +43,7 @@ from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeVectorRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
@@ -3,7 +3,7 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
@@ -14,6 +14,12 @@ from sqlalchemy.types import TypeDecorator
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
_POSTGRES_FAMILY_DIALECTS = frozenset({"postgresql", "opengauss"})
def _is_postgres_family(dialect) -> bool:
"""Return whether *dialect* preserves timezone-aware timestamps natively."""
return dialect.name in _POSTGRES_FAMILY_DIALECTS
class PortableJSON(TypeDecorator):
@@ -87,17 +93,17 @@ class BeijingDateTime(TypeDecorator):
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
value = value.replace(tzinfo=UTC)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
@@ -52,8 +52,12 @@ postgres = [
"psycopg[binary]>=3.3.3",
"psycopg-pool>=3.3.0",
]
mysql = ["asyncmy>=0.2.10"]
pymupdf = ["pymupdf4llm>=0.0.17"]
mysql = ["asyncmy>=0.2.10"]
gauss = [
"asyncpg>=0.29",
"opengauss-sqlalchemy>=2.4,<3",
]
pymupdf = ["pymupdf4llm>=0.0.17"]
[build-system]
requires = ["hatchling"]