feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
+79
@@ -0,0 +1,79 @@
|
||||
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
|
||||
|
||||
The staging index is local SQLite, never a remote provider's database. It also
|
||||
allows restoration to fetch only the vectors belonging to the current page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Records:
|
||||
def __init__(self, path: Path, kind: str, slug: str | None = None):
|
||||
self.path, self.kind, self.slug = path, kind, slug
|
||||
|
||||
def _where(self):
|
||||
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
|
||||
|
||||
def __iter__(self) -> Iterator[dict[str, Any]]:
|
||||
where, args = self._where()
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
|
||||
yield json.loads(payload)
|
||||
|
||||
def __len__(self) -> int:
|
||||
where, args = self._where()
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
|
||||
|
||||
|
||||
class PackageArchive(dict):
|
||||
def __init__(self, path: Path):
|
||||
super().__init__()
|
||||
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
|
||||
self.path = Path(temporary.name)
|
||||
temporary.close()
|
||||
try:
|
||||
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
|
||||
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
|
||||
for line_no, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
record = json.loads(line)
|
||||
kind, payload = record.get("record_type"), record.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"导出包第 {line_no} 行格式错误")
|
||||
if kind == "source":
|
||||
self["source"] = {**self.get("source", {}), **payload}
|
||||
elif kind == "graph":
|
||||
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
|
||||
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
|
||||
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
|
||||
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
|
||||
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
|
||||
else:
|
||||
raise ValueError(f"不支持的导出记录类型:{kind}")
|
||||
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
|
||||
db.commit()
|
||||
source = self.get("source", {})
|
||||
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
|
||||
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
|
||||
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
|
||||
self[key] = Records(self.path, kind)
|
||||
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def vectors_for(self, slug: str) -> Records:
|
||||
return Records(self.path, "vector", slug)
|
||||
|
||||
def close(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
@@ -76,6 +76,9 @@ class LocalWikiEmbeddingConfig(BaseModel):
|
||||
model: str = "bge-embedding-m3"
|
||||
base_url: str = ""
|
||||
api_key: str = Field(default="", repr=False)
|
||||
local_model_path: str = ""
|
||||
local_cache_dir: str = ""
|
||||
local_threads: int = Field(default=2, ge=1, le=32)
|
||||
dimensions: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
@@ -91,8 +94,8 @@ class LocalWikiEmbeddingConfig(BaseModel):
|
||||
@classmethod
|
||||
def _validate_provider(cls, value: str) -> str:
|
||||
normalized = value.strip().lower()
|
||||
if normalized not in {"openai", "openai_compatible"}:
|
||||
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
|
||||
if normalized not in {"openai", "openai_compatible", "local"}:
|
||||
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
|
||||
return normalized
|
||||
|
||||
@field_validator("base_url")
|
||||
@@ -198,7 +201,7 @@ class LocalWikiIndexConfig(BaseModel):
|
||||
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
|
||||
if self.enabled and not self.embedding.enabled:
|
||||
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
|
||||
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
|
||||
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
|
||||
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
|
||||
return self
|
||||
|
||||
|
||||
+15
@@ -501,6 +501,21 @@ class WeKnoraClient:
|
||||
raise WeKnoraError("WeKnora returned an invalid wiki graph")
|
||||
return dict(value)
|
||||
|
||||
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki generation status")
|
||||
return value
|
||||
|
||||
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return value
|
||||
|
||||
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
|
||||
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
|
||||
|
||||
async def list_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
|
||||
+8
@@ -105,4 +105,12 @@ def chunk_wiki_page(
|
||||
context.append(f"章节:{heading}")
|
||||
embedding_text = "\n".join([*context, "", chunk]).strip()
|
||||
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
|
||||
# A short article with a short summary must not disappear merely because
|
||||
# both sections fall below min_chars. Keep one combined, useful vector.
|
||||
if not result and (content or summary):
|
||||
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
|
||||
normalized = re.sub(r"\s+", " ", fallback).strip()
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
prefix = prefix_lines if title_prefix_enabled else []
|
||||
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
|
||||
return result
|
||||
+47
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
import httpx
|
||||
@@ -43,6 +44,8 @@ class StrictWikiEmbeddingClient:
|
||||
self._client = http_client
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
self._local_model = None
|
||||
self._local_lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
@@ -64,6 +67,11 @@ class StrictWikiEmbeddingClient:
|
||||
return self._client
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
if self.config.provider == "local":
|
||||
if not texts:
|
||||
return []
|
||||
async with self._local_lock:
|
||||
return await asyncio.to_thread(self._embed_local, texts)
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
@@ -76,6 +84,45 @@ class StrictWikiEmbeddingClient:
|
||||
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
|
||||
return vectors
|
||||
|
||||
def _embed_local(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled:
|
||||
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
|
||||
if self._local_model is None:
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except ImportError as exc:
|
||||
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
|
||||
TextEmbedding.add_custom_model(
|
||||
model=name,
|
||||
pooling=PoolingType.CLS,
|
||||
normalization=True,
|
||||
dim=1024,
|
||||
sources=ModelSource(hf="BAAI/bge-m3"),
|
||||
model_file="onnx/model.onnx",
|
||||
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
|
||||
license="mit",
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
|
||||
if self.config.local_model_path:
|
||||
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
|
||||
options["local_files_only"] = True
|
||||
self._local_model = TextEmbedding(**options)
|
||||
values = []
|
||||
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
|
||||
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(normalized.size)
|
||||
values.append(normalized.tolist())
|
||||
if len(values) != len(texts):
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
return values
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
+7
-2
@@ -36,8 +36,9 @@ class WikiSyncService:
|
||||
self.cache = cache
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
self.on_completed = None
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
@@ -111,6 +112,7 @@ class WikiSyncService:
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
|
||||
and not existing.get("is_remote_deleted")
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
@@ -160,7 +162,10 @@ class WikiSyncService:
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
if not failed and remote_count and notify and self.on_completed:
|
||||
result["assistant_sync"] = await self.on_completed(mapping)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
|
||||
+1
@@ -65,6 +65,7 @@ class AssistantKnowledgeImportJobRow(Base):
|
||||
|
||||
class AssistantKnowledgeWikiPageRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_pages"
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
|
||||
+136
-61
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import difflib
|
||||
@@ -60,6 +61,12 @@ def _dict(row: Any) -> dict[str, Any]:
|
||||
data["base_kind"] = data.get("base_kind") or "custom"
|
||||
data["is_global"] = data["base_kind"] == "global"
|
||||
data["description"] = data.get("description") or ""
|
||||
if isinstance(row, AssistantKnowledgeWikiPageRow):
|
||||
metadata = data.get("metadata") or {}
|
||||
for key in ("aliases", "tags", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "origins"):
|
||||
data[key] = metadata.get(key, [] if key not in {"parent_slug", "wiki_path"} else None)
|
||||
data["tags"] = metadata.get("tags") or (metadata.get("page_metadata") or {}).get("tags") or []
|
||||
data["content"] = data.get("content_markdown", "")
|
||||
return data
|
||||
|
||||
|
||||
@@ -95,7 +102,11 @@ def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
|
||||
try:
|
||||
decoded = base64.b64decode(str(blob64), validate=True)
|
||||
vector = decode_vector(decoded, int(dimensions))
|
||||
return encode_vector(vector)
|
||||
# Our packages contain normalized float32 bytes. Preserve those
|
||||
# bytes exactly: another normalization causes round-trip drift.
|
||||
if not np.isclose(np.linalg.norm(vector), 1.0, rtol=1e-5, atol=1e-6):
|
||||
return None
|
||||
return decoded, int(dimensions)
|
||||
except (ValueError, TypeError, binascii.Error):
|
||||
return None
|
||||
value = _first_vector_value(raw)
|
||||
@@ -115,6 +126,66 @@ class AssistantKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def iter_export_records(self, base_id: str):
|
||||
"""Export current Wiki articles and their original vectors, without encoding."""
|
||||
base = await self.get_base(base_id)
|
||||
if not base:
|
||||
raise ValueError("助手知识库不存在")
|
||||
yield "source", {"package_version": 1, "provider": "assistant", "type": "assistant", "knowledge_base_id": base_id, "knowledge_base_name": base["name"], "export_format": "assistant-knowledge-jsonl"}
|
||||
async with self._sf() as session:
|
||||
pages = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base_id, AssistantKnowledgeWikiPageRow.status == "active", AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
stream = await session.stream_scalars(pages.execution_options(yield_per=100))
|
||||
async for page in stream:
|
||||
data = _dict(page)
|
||||
yield "wiki_page", {**(page.metadata_json or {}), **{k: data[k] for k in ("slug", "title", "page_type", "summary", "content")}, "status": "published"}
|
||||
vectors = (
|
||||
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.join(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id)
|
||||
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
|
||||
.where(
|
||||
AssistantKnowledgeVectorRow.base_id == base_id,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
AssistantKnowledgeWikiPageRow.page_type != "chunk",
|
||||
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
|
||||
)
|
||||
)
|
||||
stream = await session.stream(vectors.execution_options(yield_per=100))
|
||||
async for vector, slug in stream:
|
||||
yield (
|
||||
"vector",
|
||||
{
|
||||
"wiki_slug": slug,
|
||||
"section_index": vector.section_index,
|
||||
"heading": vector.heading,
|
||||
"section_content": vector.section_content,
|
||||
"content_hash": vector.content_hash,
|
||||
"embedding_model": vector.embedding_model,
|
||||
"embedding_fingerprint": vector.embedding_fingerprint,
|
||||
"embedding_dimensions": vector.embedding_dimensions,
|
||||
"vector_blob_base64": base64.b64encode(vector.vector_blob).decode("ascii"),
|
||||
},
|
||||
)
|
||||
stream = await session.stream(
|
||||
select(AssistantKnowledgeEntityRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
|
||||
.where(AssistantKnowledgeEntityRow.base_id == base_id, AssistantKnowledgeEntityRow.status == "active")
|
||||
.execution_options(yield_per=100)
|
||||
)
|
||||
async for entity, slug in stream:
|
||||
yield "entity", {"id": entity.id, "name": entity.display_name, "type": entity.entity_type, "aliases": entity.aliases_json or [], "attributes": entity.attributes_json or {}, "source_wiki_slug": slug}
|
||||
stream = await session.stream(
|
||||
select(AssistantKnowledgeRelationRow, AssistantKnowledgeWikiPageRow.slug)
|
||||
.join(AssistantKnowledgeEntityRow, AssistantKnowledgeEntityRow.id == AssistantKnowledgeRelationRow.source_entity_id)
|
||||
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
|
||||
.where(AssistantKnowledgeRelationRow.base_id == base_id, AssistantKnowledgeRelationRow.status == "active")
|
||||
.execution_options(yield_per=100)
|
||||
)
|
||||
async for relation, slug in stream:
|
||||
yield (
|
||||
"relation",
|
||||
{"source_entity_id": relation.source_entity_id, "target_entity_id": relation.target_entity_id, "predicate": relation.predicate, "description": relation.description, "source_wiki_slug": slug},
|
||||
)
|
||||
|
||||
async def _base_with_stats(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
@@ -138,6 +209,7 @@ class AssistantKnowledgeRepository:
|
||||
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == row.id,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
AssistantKnowledgeWikiPageRow.page_type != "chunk",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
@@ -195,14 +267,7 @@ class AssistantKnowledgeRepository:
|
||||
# Backward compatibility: earlier builds had a single active row
|
||||
# and no ``base_kind``. Promote the first surviving base into the
|
||||
# global full-library aggregation base.
|
||||
row = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow)
|
||||
.where(AssistantKnowledgeBaseRow.status != "deleted")
|
||||
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
row = (await session.execute(select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status != "deleted").order_by(AssistantKnowledgeBaseRow.created_at.asc()).limit(1))).scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.base_kind = "global"
|
||||
if not row.description:
|
||||
@@ -655,7 +720,7 @@ class AssistantKnowledgeRepository:
|
||||
|
||||
entity_map: dict[str, str] = {}
|
||||
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
|
||||
raw_entities = list(package.get("entities") or [])
|
||||
raw_entities = package.get("entities") or []
|
||||
for raw in raw_entities:
|
||||
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
|
||||
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
|
||||
@@ -698,7 +763,7 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
)
|
||||
|
||||
raw_relations = list(package.get("relations") or [])
|
||||
raw_relations = package.get("relations") or []
|
||||
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
|
||||
for raw in raw_relations:
|
||||
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
|
||||
@@ -741,6 +806,10 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
|
||||
for entity, raw in entity_rows.values():
|
||||
# Provider Wiki graph nodes already have real articles. Do not
|
||||
# turn each node into a second, empty 'entity' article.
|
||||
if raw.get("source_wiki_slug"):
|
||||
continue
|
||||
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
|
||||
entity.aliases_json = aliases
|
||||
content = "\n".join(
|
||||
@@ -775,6 +844,8 @@ class AssistantKnowledgeRepository:
|
||||
entity.wiki_page_id = entity_page.id
|
||||
|
||||
for relation, raw in relation_rows.values():
|
||||
if raw.get("source_wiki_slug"):
|
||||
continue
|
||||
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
|
||||
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
|
||||
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
|
||||
@@ -796,7 +867,7 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
relation.wiki_page_id = relation_page.id
|
||||
|
||||
pages = list(package.get("wiki_pages") or [])
|
||||
pages = package.get("wiki_pages") or []
|
||||
if not pages and package.get("wiki"):
|
||||
wiki = package["wiki"]
|
||||
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
|
||||
@@ -807,7 +878,7 @@ class AssistantKnowledgeRepository:
|
||||
continue
|
||||
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
|
||||
title = str(raw.get("title") or source_name)
|
||||
content = str(raw.get("content") or raw.get("content_markdown") or "")
|
||||
content = str(raw.get("content") or raw.get("content_md") or raw.get("content_markdown") or "")
|
||||
summary = str(raw.get("summary") or "")
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
|
||||
page = (await session.execute(stmt)).scalar_one_or_none()
|
||||
@@ -850,32 +921,35 @@ class AssistantKnowledgeRepository:
|
||||
page.current_revision_id = revision.id
|
||||
page.status = "active"
|
||||
page.updated_at = now
|
||||
metadata = {key: raw[key] for key in ("aliases", "tags", "page_metadata", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "chunk_refs", "version") if key in raw}
|
||||
source_meta = package.get("source") or {}
|
||||
origin_mapping = mapping_id
|
||||
if not origin_mapping and (source_meta.get("provider") or source_meta.get("type")) == "weknora":
|
||||
origin_mapping = source_meta.get("knowledge_base_id")
|
||||
origins = [*((page.metadata_json or {}).get("origins") or []), *(raw.get("origins") or [])]
|
||||
origin = {"mapping_id": origin_mapping, "source_key": source_key, "source_name": source_name, "slug": slug}
|
||||
origins = [value for value in origins if isinstance(value, dict) and value.get("source_key") != source_key]
|
||||
metadata["origins"] = list({(v.get("mapping_id"), v.get("source_key"), v.get("slug")): v for v in [*origins, origin]}.values())
|
||||
page.metadata_json = metadata
|
||||
page_ids_by_slug[slug] = page.id
|
||||
for entity, raw_entity in entity_rows.values():
|
||||
if raw_entity.get("source_wiki_slug") == slug:
|
||||
entity.wiki_page_id = page.id
|
||||
if len(page_ids_by_slug) % 50 == 0:
|
||||
await session.flush()
|
||||
|
||||
chunks = list(package.get("chunks") or [])
|
||||
chunks = package.get("chunks") or []
|
||||
chunk_pages = 0
|
||||
imported_vectors = 0
|
||||
for index, raw in enumerate(chunks):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
content = str(
|
||||
raw.get("content")
|
||||
or raw.get("section_content")
|
||||
or raw.get("text")
|
||||
or raw.get("chunk_content")
|
||||
or ""
|
||||
).strip()
|
||||
content = str(raw.get("content") or raw.get("section_content") or raw.get("text") or raw.get("chunk_content") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
|
||||
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
|
||||
title = str(
|
||||
raw.get("title")
|
||||
or raw.get("knowledge_title")
|
||||
or raw.get("filename")
|
||||
or raw.get("knowledge_filename")
|
||||
or f"{source_name} 片段 {index + 1}"
|
||||
)
|
||||
title = str(raw.get("title") or raw.get("knowledge_title") or raw.get("filename") or raw.get("knowledge_filename") or f"{source_name} 片段 {index + 1}")
|
||||
heading = str(raw.get("heading") or raw.get("section_title") or "")
|
||||
summary = content[:240]
|
||||
page_content = "\n".join(
|
||||
@@ -919,7 +993,7 @@ class AssistantKnowledgeRepository:
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
|
||||
source_chunk_id=raw_chunk_id,
|
||||
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
|
||||
section_index=int(next((raw[key] for key in ("section_index", "chunk_index") if raw.get(key) is not None), index)),
|
||||
heading=heading or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
@@ -933,19 +1007,19 @@ class AssistantKnowledgeRepository:
|
||||
imported_vectors += 1
|
||||
chunk_pages += 1
|
||||
|
||||
for index, raw in enumerate(list(package.get("vectors") or [])):
|
||||
for index, raw in enumerate(package.get("vectors") or []):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
raise ValueError("向量记录必须为对象")
|
||||
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
|
||||
page_id = page_ids_by_slug.get(slug)
|
||||
if not page_id:
|
||||
continue
|
||||
raise ValueError(f"向量引用不存在的 Wiki:{slug}")
|
||||
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
vector_blob = _vector_blob_from_payload(raw)
|
||||
if vector_blob is None:
|
||||
continue
|
||||
raise ValueError(f"Wiki 向量格式无效:{slug}")
|
||||
blob, dimensions = vector_blob
|
||||
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
|
||||
session.add(
|
||||
@@ -957,7 +1031,7 @@ class AssistantKnowledgeRepository:
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
|
||||
source_chunk_id=raw_chunk_id or None,
|
||||
section_index=int(raw.get("section_index") or index),
|
||||
section_index=int(raw["section_index"]) if raw.get("section_index") is not None else index,
|
||||
heading=str(raw.get("heading") or "") or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
@@ -969,9 +1043,11 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
)
|
||||
imported_vectors += 1
|
||||
if imported_vectors % 50 == 0:
|
||||
await session.flush()
|
||||
|
||||
counts = {
|
||||
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
|
||||
"wiki_pages": len(pages) + sum(not raw.get("source_wiki_slug") for _, raw in entity_rows.values()) + sum(not raw.get("source_wiki_slug") for _, raw in relation_rows.values()),
|
||||
"documents": len(package.get("documents") or []),
|
||||
"chunks": len(chunks),
|
||||
"chunk_pages": chunk_pages,
|
||||
@@ -1008,6 +1084,7 @@ class AssistantKnowledgeRepository:
|
||||
query: str | None = None,
|
||||
page_type: str | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
include_chunks: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
@@ -1024,7 +1101,7 @@ class AssistantKnowledgeRepository:
|
||||
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
|
||||
if page_type:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.slug, AssistantKnowledgeWikiPageRow.id).offset(offset).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
@@ -1070,10 +1147,7 @@ class AssistantKnowledgeRepository:
|
||||
or_(
|
||||
AssistantKnowledgeWikiPageRow.title.ilike(needle),
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
|
||||
*[
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
|
||||
for token in tokens[:8]
|
||||
],
|
||||
*[AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%") for token in tokens[:8]],
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
|
||||
@@ -1098,6 +1172,7 @@ class AssistantKnowledgeRepository:
|
||||
results = rows[:limit]
|
||||
for row in results:
|
||||
row["score"] = score(row)
|
||||
row["retrieval_mode"] = "keyword"
|
||||
return results
|
||||
|
||||
async def _search_pages_by_vector(
|
||||
@@ -1119,31 +1194,36 @@ class AssistantKnowledgeRepository:
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
|
||||
)
|
||||
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
|
||||
.where(
|
||||
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
|
||||
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
|
||||
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
|
||||
AssistantKnowledgeVectorRow.embedding_fingerprint == embedding_client.fingerprint,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
|
||||
)
|
||||
if not include_chunks:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
async with self._sf() as session:
|
||||
rows = list((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return []
|
||||
tokens = [token for token in query.strip().lower().split() if token]
|
||||
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
|
||||
for vector, page in rows:
|
||||
try:
|
||||
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
|
||||
except ValueError:
|
||||
continue
|
||||
score = float(np.dot(query_vector, stored))
|
||||
if tokens:
|
||||
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
|
||||
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
|
||||
ranked.append((score, vector, page))
|
||||
# Bounded scan: keep only the best section per page, and at most limit
|
||||
# candidates. Never load a multi-GB vector library into one matrix.
|
||||
async with self._sf() as session:
|
||||
stream = await session.stream(stmt.execution_options(yield_per=256))
|
||||
async for batch in stream.partitions(256):
|
||||
|
||||
def score_batch():
|
||||
values = []
|
||||
for vector, page in batch:
|
||||
values.append((float(np.dot(query_vector, decode_vector(vector.vector_blob, dimensions))), vector, page))
|
||||
return values
|
||||
|
||||
candidates = ranked + await asyncio.to_thread(score_batch)
|
||||
best = {}
|
||||
for item in sorted(candidates, key=lambda item: item[0], reverse=True):
|
||||
best.setdefault(item[2].id, item)
|
||||
ranked = list(best.values())[:limit]
|
||||
ranked.sort(key=lambda item: item[0], reverse=True)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
@@ -1320,12 +1400,7 @@ class AssistantKnowledgeRepository:
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(AssistantKnowledgeImportJobRow)
|
||||
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
|
||||
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
stmt = select(AssistantKnowledgeImportJobRow).where(AssistantKnowledgeImportJobRow.base_id == base["id"]).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return dict(row) if row else None
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
|
||||
+4
-3
@@ -41,10 +41,11 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
return row
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
result = (await session.execute(stmt)).one_or_none()
|
||||
return {**_dict(result[0]), "vector_count": result[1]} if result else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""Preserve source Wiki metadata during package round trips."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260903_01"
|
||||
down_revision = ("20260902_01", "20260831_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
columns = {col["name"] for col in sa.inspect(op.get_bind()).get_columns("assistant_knowledge_wiki_pages")}
|
||||
if "metadata_json" not in columns:
|
||||
op.add_column("assistant_knowledge_wiki_pages", sa.Column("metadata_json", PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("assistant_knowledge_wiki_pages", "metadata_json")
|
||||
Reference in new issue
Block a user