feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -65,6 +65,7 @@ class AssistantKnowledgeImportJobRow(Base):
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import base64
import binascii
import difflib
@@ -60,6 +61,12 @@ def _dict(row: Any) -> dict[str, Any]:
data["base_kind"] = data.get("base_kind") or "custom"
data["is_global"] = data["base_kind"] == "global"
data["description"] = data.get("description") or ""
if isinstance(row, AssistantKnowledgeWikiPageRow):
metadata = data.get("metadata") or {}
for key in ("aliases", "tags", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "origins"):
data[key] = metadata.get(key, [] if key not in {"parent_slug", "wiki_path"} else None)
data["tags"] = metadata.get("tags") or (metadata.get("page_metadata") or {}).get("tags") or []
data["content"] = data.get("content_markdown", "")
return data
@@ -95,7 +102,11 @@ def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
try:
decoded = base64.b64decode(str(blob64), validate=True)
vector = decode_vector(decoded, int(dimensions))
return encode_vector(vector)
# Our packages contain normalized float32 bytes. Preserve those
# bytes exactly: another normalization causes round-trip drift.
if not np.isclose(np.linalg.norm(vector), 1.0, rtol=1e-5, atol=1e-6):
return None
return decoded, int(dimensions)
except (ValueError, TypeError, binascii.Error):
return None
value = _first_vector_value(raw)
@@ -115,6 +126,66 @@ class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def iter_export_records(self, base_id: str):
"""Export current Wiki articles and their original vectors, without encoding."""
base = await self.get_base(base_id)
if not base:
raise ValueError("助手知识库不存在")
yield "source", {"package_version": 1, "provider": "assistant", "type": "assistant", "knowledge_base_id": base_id, "knowledge_base_name": base["name"], "export_format": "assistant-knowledge-jsonl"}
async with self._sf() as session:
pages = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base_id, AssistantKnowledgeWikiPageRow.status == "active", AssistantKnowledgeWikiPageRow.page_type != "chunk")
stream = await session.stream_scalars(pages.execution_options(yield_per=100))
async for page in stream:
data = _dict(page)
yield "wiki_page", {**(page.metadata_json or {}), **{k: data[k] for k in ("slug", "title", "page_type", "summary", "content")}, "status": "published"}
vectors = (
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow.slug)
.join(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id)
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
.where(
AssistantKnowledgeVectorRow.base_id == base_id,
AssistantKnowledgeWikiPageRow.status == "active",
AssistantKnowledgeWikiPageRow.page_type != "chunk",
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
)
)
stream = await session.stream(vectors.execution_options(yield_per=100))
async for vector, slug in stream:
yield (
"vector",
{
"wiki_slug": slug,
"section_index": vector.section_index,
"heading": vector.heading,
"section_content": vector.section_content,
"content_hash": vector.content_hash,
"embedding_model": vector.embedding_model,
"embedding_fingerprint": vector.embedding_fingerprint,
"embedding_dimensions": vector.embedding_dimensions,
"vector_blob_base64": base64.b64encode(vector.vector_blob).decode("ascii"),
},
)
stream = await session.stream(
select(AssistantKnowledgeEntityRow, AssistantKnowledgeWikiPageRow.slug)
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
.where(AssistantKnowledgeEntityRow.base_id == base_id, AssistantKnowledgeEntityRow.status == "active")
.execution_options(yield_per=100)
)
async for entity, slug in stream:
yield "entity", {"id": entity.id, "name": entity.display_name, "type": entity.entity_type, "aliases": entity.aliases_json or [], "attributes": entity.attributes_json or {}, "source_wiki_slug": slug}
stream = await session.stream(
select(AssistantKnowledgeRelationRow, AssistantKnowledgeWikiPageRow.slug)
.join(AssistantKnowledgeEntityRow, AssistantKnowledgeEntityRow.id == AssistantKnowledgeRelationRow.source_entity_id)
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
.where(AssistantKnowledgeRelationRow.base_id == base_id, AssistantKnowledgeRelationRow.status == "active")
.execution_options(yield_per=100)
)
async for relation, slug in stream:
yield (
"relation",
{"source_entity_id": relation.source_entity_id, "target_entity_id": relation.target_entity_id, "predicate": relation.predicate, "description": relation.description, "source_wiki_slug": slug},
)
async def _base_with_stats(
self,
session: AsyncSession,
@@ -138,6 +209,7 @@ class AssistantKnowledgeRepository:
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
AssistantKnowledgeWikiPageRow.base_id == row.id,
AssistantKnowledgeWikiPageRow.status == "active",
AssistantKnowledgeWikiPageRow.page_type != "chunk",
)
)
).scalar_one()
@@ -195,14 +267,7 @@ class AssistantKnowledgeRepository:
# Backward compatibility: earlier builds had a single active row
# and no ``base_kind``. Promote the first surviving base into the
# global full-library aggregation base.
row = (
await session.execute(
select(AssistantKnowledgeBaseRow)
.where(AssistantKnowledgeBaseRow.status != "deleted")
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
.limit(1)
)
).scalar_one_or_none()
row = (await session.execute(select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status != "deleted").order_by(AssistantKnowledgeBaseRow.created_at.asc()).limit(1))).scalar_one_or_none()
if row is not None:
row.base_kind = "global"
if not row.description:
@@ -655,7 +720,7 @@ class AssistantKnowledgeRepository:
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
raw_entities = list(package.get("entities") or [])
raw_entities = package.get("entities") or []
for raw in raw_entities:
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
@@ -698,7 +763,7 @@ class AssistantKnowledgeRepository:
)
)
raw_relations = list(package.get("relations") or [])
raw_relations = package.get("relations") or []
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
for raw in raw_relations:
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
@@ -741,6 +806,10 @@ class AssistantKnowledgeRepository:
)
for entity, raw in entity_rows.values():
# Provider Wiki graph nodes already have real articles. Do not
# turn each node into a second, empty 'entity' article.
if raw.get("source_wiki_slug"):
continue
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
entity.aliases_json = aliases
content = "\n".join(
@@ -775,6 +844,8 @@ class AssistantKnowledgeRepository:
entity.wiki_page_id = entity_page.id
for relation, raw in relation_rows.values():
if raw.get("source_wiki_slug"):
continue
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
@@ -796,7 +867,7 @@ class AssistantKnowledgeRepository:
)
relation.wiki_page_id = relation_page.id
pages = list(package.get("wiki_pages") or [])
pages = package.get("wiki_pages") or []
if not pages and package.get("wiki"):
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
@@ -807,7 +878,7 @@ class AssistantKnowledgeRepository:
continue
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
title = str(raw.get("title") or source_name)
content = str(raw.get("content") or raw.get("content_markdown") or "")
content = str(raw.get("content") or raw.get("content_md") or raw.get("content_markdown") or "")
summary = str(raw.get("summary") or "")
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
page = (await session.execute(stmt)).scalar_one_or_none()
@@ -850,32 +921,35 @@ class AssistantKnowledgeRepository:
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
metadata = {key: raw[key] for key in ("aliases", "tags", "page_metadata", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "chunk_refs", "version") if key in raw}
source_meta = package.get("source") or {}
origin_mapping = mapping_id
if not origin_mapping and (source_meta.get("provider") or source_meta.get("type")) == "weknora":
origin_mapping = source_meta.get("knowledge_base_id")
origins = [*((page.metadata_json or {}).get("origins") or []), *(raw.get("origins") or [])]
origin = {"mapping_id": origin_mapping, "source_key": source_key, "source_name": source_name, "slug": slug}
origins = [value for value in origins if isinstance(value, dict) and value.get("source_key") != source_key]
metadata["origins"] = list({(v.get("mapping_id"), v.get("source_key"), v.get("slug")): v for v in [*origins, origin]}.values())
page.metadata_json = metadata
page_ids_by_slug[slug] = page.id
for entity, raw_entity in entity_rows.values():
if raw_entity.get("source_wiki_slug") == slug:
entity.wiki_page_id = page.id
if len(page_ids_by_slug) % 50 == 0:
await session.flush()
chunks = list(package.get("chunks") or [])
chunks = package.get("chunks") or []
chunk_pages = 0
imported_vectors = 0
for index, raw in enumerate(chunks):
if not isinstance(raw, dict):
continue
content = str(
raw.get("content")
or raw.get("section_content")
or raw.get("text")
or raw.get("chunk_content")
or ""
).strip()
content = str(raw.get("content") or raw.get("section_content") or raw.get("text") or raw.get("chunk_content") or "").strip()
if not content:
continue
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
title = str(
raw.get("title")
or raw.get("knowledge_title")
or raw.get("filename")
or raw.get("knowledge_filename")
or f"{source_name} 片段 {index + 1}"
)
title = str(raw.get("title") or raw.get("knowledge_title") or raw.get("filename") or raw.get("knowledge_filename") or f"{source_name} 片段 {index + 1}")
heading = str(raw.get("heading") or raw.get("section_title") or "")
summary = content[:240]
page_content = "\n".join(
@@ -919,7 +993,7 @@ class AssistantKnowledgeRepository:
import_job_id=job.id,
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
source_chunk_id=raw_chunk_id,
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
section_index=int(next((raw[key] for key in ("section_index", "chunk_index") if raw.get(key) is not None), index)),
heading=heading or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
@@ -933,19 +1007,19 @@ class AssistantKnowledgeRepository:
imported_vectors += 1
chunk_pages += 1
for index, raw in enumerate(list(package.get("vectors") or [])):
for index, raw in enumerate(package.get("vectors") or []):
if not isinstance(raw, dict):
continue
raise ValueError("向量记录必须为对象")
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
page_id = page_ids_by_slug.get(slug)
if not page_id:
continue
raise ValueError(f"向量引用不存在的 Wiki:{slug}")
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
if not content:
continue
vector_blob = _vector_blob_from_payload(raw)
if vector_blob is None:
continue
raise ValueError(f"Wiki 向量格式无效:{slug}")
blob, dimensions = vector_blob
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
session.add(
@@ -957,7 +1031,7 @@ class AssistantKnowledgeRepository:
import_job_id=job.id,
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
source_chunk_id=raw_chunk_id or None,
section_index=int(raw.get("section_index") or index),
section_index=int(raw["section_index"]) if raw.get("section_index") is not None else index,
heading=str(raw.get("heading") or "") or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
@@ -969,9 +1043,11 @@ class AssistantKnowledgeRepository:
)
)
imported_vectors += 1
if imported_vectors % 50 == 0:
await session.flush()
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"wiki_pages": len(pages) + sum(not raw.get("source_wiki_slug") for _, raw in entity_rows.values()) + sum(not raw.get("source_wiki_slug") for _, raw in relation_rows.values()),
"documents": len(package.get("documents") or []),
"chunks": len(chunks),
"chunk_pages": chunk_pages,
@@ -1008,6 +1084,7 @@ class AssistantKnowledgeRepository:
query: str | None = None,
page_type: str | None = None,
limit: int = 100,
offset: int = 0,
include_chunks: bool = False,
) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
@@ -1024,7 +1101,7 @@ class AssistantKnowledgeRepository:
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
if page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.slug, AssistantKnowledgeWikiPageRow.id).offset(offset).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
@@ -1070,10 +1147,7 @@ class AssistantKnowledgeRepository:
or_(
AssistantKnowledgeWikiPageRow.title.ilike(needle),
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
*[
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
for token in tokens[:8]
],
*[AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%") for token in tokens[:8]],
)
)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
@@ -1098,6 +1172,7 @@ class AssistantKnowledgeRepository:
results = rows[:limit]
for row in results:
row["score"] = score(row)
row["retrieval_mode"] = "keyword"
return results
async def _search_pages_by_vector(
@@ -1119,31 +1194,36 @@ class AssistantKnowledgeRepository:
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
)
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
.where(
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
AssistantKnowledgeVectorRow.embedding_fingerprint == embedding_client.fingerprint,
AssistantKnowledgeWikiPageRow.status == "active",
)
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
)
if not include_chunks:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
async with self._sf() as session:
rows = list((await session.execute(stmt)).all())
if not rows:
return []
tokens = [token for token in query.strip().lower().split() if token]
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
for vector, page in rows:
try:
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
except ValueError:
continue
score = float(np.dot(query_vector, stored))
if tokens:
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
ranked.append((score, vector, page))
# Bounded scan: keep only the best section per page, and at most limit
# candidates. Never load a multi-GB vector library into one matrix.
async with self._sf() as session:
stream = await session.stream(stmt.execution_options(yield_per=256))
async for batch in stream.partitions(256):
def score_batch():
values = []
for vector, page in batch:
values.append((float(np.dot(query_vector, decode_vector(vector.vector_blob, dimensions))), vector, page))
return values
candidates = ranked + await asyncio.to_thread(score_batch)
best = {}
for item in sorted(candidates, key=lambda item: item[0], reverse=True):
best.setdefault(item[2].id, item)
ranked = list(best.values())[:limit]
ranked.sort(key=lambda item: item[0], reverse=True)
results: list[dict[str, Any]] = []
@@ -1320,12 +1400,7 @@ class AssistantKnowledgeRepository:
if base is None:
return []
async with self._sf() as session:
stmt = (
select(AssistantKnowledgeImportJobRow)
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
.limit(limit)
)
stmt = select(AssistantKnowledgeImportJobRow).where(AssistantKnowledgeImportJobRow.base_id == base["id"]).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None: